#!/bin/sh
# This is a shell archive, meaning:
# 1. Remove everything above the #!/bin/sh line.
# 2. Save the resulting text in a file.
# 3. Execute the file with /bin/sh (not csh) to create the files:
#	p4-1.2
# This archive created: Wed Nov 18 15:44:41 1992
export PATH; PATH=/bin:$PATH
if test ! -d 'p4-1.2'
then
	mkdir 'p4-1.2'
fi
cd 'p4-1.2'
if test -f 'Makefile'
then
	echo shar: over-writing existing file "'Makefile'"
fi
cat << \SHAR_EOF > 'Makefile'
# top level makefile for p4

SHELL=/bin/sh

MFLAGS = 
#MFLAGS = -i    # for alliant fx/8
  
P4_DIRS = util lib alog usc include bin servers 

EXAMPLES_DIRS = contrib messages monitors

ALL_DIRS = $(P4_DIRS) $(EXAMPLES_DIRS) lib_f messages_f contrib_f doc usc/usctest

DIRS = $(ALL_DIRS)

RM = /bin/rm

all:
	@echo making p4 for machine=$(MACHINE)
	@make makefiles MACHINE=$(MACHINE)
	@make $(MFLAGS) links MACHINE=$(MACHINE)
	@for dir in $(DIRS) ;\
	  do \
	  echo " " ;\
	  echo making p4 in directory $$dir ;\
	  (cd $$dir; make $(MFLAGS) default) ;\
	  done 

p4:
	@echo making p4 for machine=$(MACHINE)
	@make makefiles MACHINE=$(MACHINE)
	@make $(MFLAGS) links MACHINE=$(MACHINE)
	@for dir in $(P4_DIRS) ;\
	  do \
	  echo " " ;\
	  echo making p4 in directory $$dir ;\
	  (cd $$dir; make $(MFLAGS) default) ;\
	  done 

makefiles:
	@(cd util; make -f makefile.proto defs.MD MACHINE=$(MACHINE))
	@echo "making Makefiles in " $(DIRS)
	@for dir in $(DIRS) ;\
	  do \
	  cat util/defs.MD $$dir/makefile.proto > $$dir/Makefile ;\
	  done 

#  @echo "#define  P4_$_MACHINE_TYPE \"$(MACHINE)\"" >> lib/p4_config.h
config_hdr:
	@echo "#define P4_COMPILED_TIME  \"`date`\"" > lib/p4_config.h
	@echo "#define  P4_MACHINE_TYPE \"$(MACHINE)\"" >> lib/p4_config.h
	@echo "#ifndef  $(MACHINE)" >> lib/p4_config.h
	@echo "#define  $(MACHINE)" >> lib/p4_config.h
	@echo "#endif  "            >> lib/p4_config.h
	@cat OPTIONS >> lib/p4_config.h

links:
	@if [ -f lib/alog.h ] ;\
	then\
	    true ;\
	else\
	    ln alog/alog*.h lib ;\
	    ln alog/alog*.h include ;\
	fi
	@if [ -f lib/usc.h ] ;\
	then\
	    true ;\
	else\
	    ln usc/usc*.h lib ;\
	    ln usc/usc*.h include ;\
	fi
	@if [ -f lib/p4_config.h ] ;\
	then\
	    true ;\
	else\
	    make config_hdr MACHINE=$(MACHINE) ;\
	fi
	@if [ -f include/p4.h ] ;\
	then\
	    true ;\
	else\
	    ln lib/p4*.h include ;\
	fi
	@if [ -f messages_f/p4f.h ] ;\
	then\
	    true ;\
	else\
	    ln lib_f/p4f.h messages_f ;\
	fi
	@if [ -f contrib_f/p4f.h ] ;\
	then\
	    true ;\
	else\
	    ln lib_f/p4f.h contrib_f ;\
	fi

clean:  
	@for dir in $(DIRS) ;\
	  do \
	  echo cleaning $$dir ;\
	  (cd $$dir; make clean) ;\
	  done
	$(RM) -f *~ *.o *.bak

dist:	realclean   

realclean:
	$(RM) -f *~ *.o *.bak
	@for dir in $(DIRS) ;\
	  do \
	  echo cleaning $$dir ;\
	  (cd $$dir; make realclean) ;\
	  done

Makefiles: makefiles

INSTALLDIR = ./$(MACHINE)
install:
	@echo installing p4 in $(INSTALLDIR)
	@if [ -d $(INSTALLDIR) ] ;\
	then\
	    echo 'using existing directory $(INSTALLDIR)' ;\
	else\
	    mkdir $(INSTALLDIR) ;\
	fi
	@cp Makefile $(INSTALLDIR)
	@mkdir $(INSTALLDIR)/util
	@cp util/* $(INSTALLDIR)/util
	@mkdir $(INSTALLDIR)/include
	@cp include/*.h $(INSTALLDIR)/include
	@mkdir $(INSTALLDIR)/lib
	@cp lib/libp4.a lib/p4_cmain.o $(INSTALLDIR)/lib
	@grep RANLIB util/defs.MD > $(INSTALLDIR)/lib/makeranlib
	@echo "doran:" >> $(INSTALLDIR)/lib/makeranlib
	@echo "	$$(RANLIB) libp4.a" >> $(INSTALLDIR)/lib/makeranlib
	@(cd  $(INSTALLDIR)/lib; make -f makeranlib doran; $(RM) makeranlib)
	@mkdir $(INSTALLDIR)/lib_f
	@cp lib_f/libp4_f.a lib_f/p4_fmain.o lib_f/*.h $(INSTALLDIR)/lib_f
	@grep RANLIB util/defs.MD > $(INSTALLDIR)/lib_f/makeranlib
	@echo "doran:" >> $(INSTALLDIR)/lib_f/makeranlib
	@echo "	$$(RANLIB) libp4_f.a" >> $(INSTALLDIR)/lib_f/makeranlib
	@(cd  $(INSTALLDIR)/lib_f; make -f makeranlib doran; $(RM) makeranlib)
	@mkdir $(INSTALLDIR)/bin
	@cp bin/* $(INSTALLDIR)/bin
	@mkdir $(INSTALLDIR)/examples
	@cp messages/sr_test.c messages/sr_user.h  \
	  messages/systest.c messages/makefile.proto \
	  messages/runcube messages/rundelta \
	  messages/procgroup $(INSTALLDIR)/examples
	@sed -e "s@P4_HOME_DIR = ..@P4_HOME_DIR = $(INSTALLDIR)@" \
	  $(INSTALLDIR)/examples/makefile.proto >  $(INSTALLDIR)/temp
	@mv $(INSTALLDIR)/temp  $(INSTALLDIR)/examples/makefile.proto
	@make makefiles DIRS=$(INSTALLDIR)/examples
	@(cd $(INSTALLDIR)/examples; make sr_test systest)
	@mkdir $(INSTALLDIR)/examples_f
	@cp messages_f/systest.f messages_f/p4f.h  \
	  messages_f/makefile.proto \
	  messages_f/procgroup $(INSTALLDIR)/examples_f
	@sed -e "s@P4_HOME_DIR = ..@P4_HOME_DIR = $(INSTALLDIR)@" \
	  $(INSTALLDIR)/examples_f/makefile.proto >  $(INSTALLDIR)/temp
	@mv $(INSTALLDIR)/temp  $(INSTALLDIR)/examples_f/makefile.proto
	@make makefiles DIRS=$(INSTALLDIR)/examples_f
	@(cd $(INSTALLDIR)/examples_f; make systest)
	@echo installed p4 in $(INSTALLDIR)
SHAR_EOF
if test -f 'OPTIONS'
then
	echo shar: over-writing existing file "'OPTIONS'"
fi
cat << \SHAR_EOF > 'OPTIONS'
/* #define ALOG_TRACE */
/* #define SYSV_IPC */
#define P4_DPRINTFL
SHAR_EOF
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'


		      The P4 Parallel Processing Library

p4 is a package of procedures that support development of parallel programs
that are portable across a wide variety of architectures.  For the
shared-memory model of parallel computation, p4 provides a set of primitives
from which monitors can be constructed, as well as a set of useful monitors.
For the distributed-memory model, p4 provides typed send and receive
operations, and creation of processes according to a text file describing
files to be executed by created processes and the machines to execute them
on.  These two models can be combined to form the "cluster" model.

Authorship:  This system owes much to many people (see History, below) but the
current version was written by Ralph Butler of the University of North Florida
and Argonne National Laboratory, and Rusty Lusk at Argonne National Laboratory.

History: The system is directly decended from the "Argonne macros", or
"monmacs", or "parmacs", the system built at Argonne by Rusty Lusk and Ross
Overbeek using m4 macros, and documented in the book "Portable Programs for
Parallel Processors", by Boyle, Butler, Disz, Glickfeld, Lusk, Overbeek,
Patterson, and Stevens, all of whom contributed to that version of the system.
Major contributions were also made by Bob Olson, now at the University of
Illinois, and Remy Evard, now at the University of Oregon.  The book is
published by Holt, Rinehart, and Winston (ISBN 0-03-014404-3).  p4 does not
change any of the basic concepts presented there, but is a completely new
implementation.  Major differences are:

  The m4 macro processor is not part of the system; it is now constructed 
    entirely with C macros and functions.
  Processes on various machines are specified in a text file that is read at
    run time, rather than in macros that are compiled into the executable
    file.
  More machines are supported, notably the BBN Butterflies, the Intel
    IPSC/860, and the IBM RS/6000.
  Machine dependent code has been isolated in two files, making the system
    easier to port.


The files and subdirectories in this directory are:

  README     - this file
  alog       - the logging package for debugging/analyzing p4 pgms
  disclaimer - the "no warranty" statement
  doc	     - a draft user manual, man page, and other documentation
  lib	     - the p4 system C source code
  lib_f	     - Fortran interface routines
  makefile   - the top-level makefile for installing p4
  messages   - distributed-memory examples
  messages_f - distributed-memory examples in Fortran
  monitors   - shared-memory examples
  contrib    - miscellaneous other examples
  contrib_f  - miscellaneous fortran examples
  util	     - installation files, especially defs.all and makefile.proto
  bin	     - a few shell scripts and executable version of server daemon(s)

See the manual in the doc subdirectory for full details on how to install
and use p4.  Following is a brief description of how to do the
installation.


To build p4, position yourself in this directory and type:

   make all MACHINE=<machine>

where <machine> is one of the machine names listed in util/machines.

For example:

  make all MACHINE=SYMMETRY


The 'all' is optional, for example

  make MACHINE=IPSC860_HOST


To save disk space, various intermediate files can be removed with

  make clean


The system can be restored to its original, machine-independent state with

  make realclean

Note that this removes the machine-dependent Makefiles in each directory, so
the operation is not idempotent.

It is also possible to install (or clean) only some of the directories:

  make all MACHINE=SUN DIRS=messages

  make clean DIRS='monitors messages'


To install only the Makefiles in all subdirectories, use:

  make makefiles MACHINE=<machine>

You can have p4 build a makefile in your own directory by copying one of
the makefile.proto files from an example directory (e.g. messages or
monitors).  Then edit the proto file to include information about your
own programs and the location of the p4 library (LIBDIR).
Finally, you build the Makefile by getting into the 
top-level p4 directory and typing:

    make makefiles MACHINE=<machine> DIRS=<your_full_pathname>

To add a new machine, or to change the characteristic parameters associated
with an existing one, edit the file util/defs.all

To add a new example program in one of the application directories, edit the
file makefile.proto in the appropriate directory.

p4 currently traps the signals SIGINT and SIGUSR1.  The SIGINT interrupt is
trapped in order to do some cleanup especially on System V machines where
IPC facilities must be returned to the OS.  If the user decides to trap
this interrupt for his own cleanup, he should invoke p4_error as part of
any termination processing.  p4 uses the SIGUSR1 interrupt internally by a
listener process for socket communications.  The only time in which p4
would not need this interrupt would be in those instances in which no
socket communications are to be performed.



/******************  OLD STUFF  *****************************************/


Status of version 0.2 (2/1/1992):
  
  As in version 0.1 (see below), processes on the Butterfly may not be 
    created in the cluster you are expecting.  In particular, processes
    which are started via remote shell from another machine may start in the
    public cluster.  We will explore this further as time permits.
  
  Also on the butterfly, we have implemented quite a bit of code for the 
    TCMP package planned for that machine, but do not yet have it stable.

  Shared memory operations in Fortran are not supported on the Sequents.
    This implies that you cannot have a procgroup file entry such as 
    "local 2" for a Fortran program on the Sequent, because such an entry
    would imply that messages would be passed thru shared memory.

  We do not currently support System V IPC on any machine although we are
    exploring it.  The major reason that we have not supported it is that
    in some instances the operating system may not reclaim IPC resources if a
    program abnormally terminates, and the user must explicitly do it.  We 
    are planning to make p4 handle such cases when possible.
  
  The alog/upshot packages are available as graphical debugging tools.  In
    the near future, we hope to instrument the internals of p4 with logging
    calls for these packages.
  
  The new machine type IBM_3090_AIX does not support Fortran at this point.
    We have encountered one or two "standard" Fortran functions such as getarg
    that are not available on the machine which are using.  We will explore
    this problem more as time permits.

  The Fortran on the NeXT has the same problem as the IBM_3090_AIX.

  The Fortran on the Titan is not yet working correctly.

  If your machine is running an old socket implementation (e.g. 4.2),
    then you may experience very slow communications via sockets.
    Also, in the current release, we are only using the socket option
    TCP_NODELAY on BSD systems to boost performance.  We have found
    that the setsockopt call is not available on some SYSV machines 
    that we have ported to; we plan to fix this for those SYSV
    environments that do support it in the future.  If you wish to 
    turn that option on, the relevant code is protected by ifdef P4BSD
    lines in p4_sock_util.h and can easily be changed.
  
  On an SGI machine, the make facility will only work correctly if you
    set your environment variable SHELL to the Bourne shell, since our 
    makefiles contain Bourne shell code at this point.



Status of version 0.1 (2/1/1991):

  The C version is stable.  No significant changes are planned.

  The examples all work, but could stand some cleaning up.  Subsequent
    releases may have different examples.  The grid example in the messages
    directory has a compile-time configuration of 4 processes.

  The p4_logging routines have not been tested.  They are in the process of
    being replaced by the atrace package.

  The Fortran version is included, but it is not guaranteed to compile and
    link on any of the given machines without local modifications.  It is
    really a place-holder for a new version to come real soon now, which may
    have different procedure names and parameters but the same idea.

  The tutorial is in very rough draft form.  It is expected to be extended in
    subsequent releases.

  It is not possible to create node processes on the IPSC860 from machines
    other than the IPSC host machine, nor is it possible to create node
    processes excuting different files.  The only model that works on this
    machine has a master on the srm and the same executable file loaded on all
    the nodes.  We hope to change this restriction soon.

  The clock on the ATT_3B2 is coarse-grained.

  The socket-pair system routine on the 3B2 has a bug in it which causes an
    error message to be reported at the end of a run.  It should not effect
    the job as it is running.

  The Butterflies are treated as shared-memory machines, with primitive memory
    management.  The next release should have BBN native memory management 
    with support for interleaved memory, and an optimized message-passing
    system written by Ken Sedgewick.

  On the butterfly be alert to the fact that processes may not be created in
    the cluster you were expecting.  Automatic cluster management is being
    explored.

  On the IPSC860, mixing p4 message passing with Intel message passing, even
    the system messages that are associated with global data operations, may
    lead to unexpected (and bad) results.  We are working on this now.
SHAR_EOF
if test ! -d 'alog'
then
	mkdir 'alog'
fi
cd 'alog'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'


			Preliminary ALOG package


Here are the files necessary to log events from parallel programs running
on any of the following machines: SUN, NEXT, IBM3090, RS6000, SYMMETRY,
BALANCE, MULTIMAX, BFLY1, BFLY2, BFLY2_TCMP, IPSC860_HOST, IPSC860_NODE,
DELTA, TITAN, SGI.

Both C macros and Fortran-callable routines are included.

Log files created during execution are merged with the 'mergelogs' program 
and viewed with upshot.  Upshot now supports state duration histograms and 
a primitive animation facility (Edgewise upshot).

An example will be included soon.
SHAR_EOF
if test -f 'README_ALOG'
then
	echo shar: over-writing existing file "'README_ALOG'"
fi
cat << \SHAR_EOF > 'README_ALOG'
 --------------------------------------------------------------------
 | README (Summary documentation for the ALOG Tracing package)      |
 |                                                                  |
 | This file briefly describes the functionality  and the user      |
 | interface for the ALOG system.                                   |
 |                                                                  |
 | Written by:  Arun Nanda    (08/15/91)                            |
 |                                                                  |
 --------------------------------------------------------------------

This directory contains the source file for the tracing system.
The code presented here can be transported to any of the following 
computers:

                ALLIANT
                BALANCE
                BFLY1
                BFLY2
                BFLY2_TCMP
                DELTA
                IBM3090
                RS6000
                IPSC860_HOST
                IPSC860_NODE
                MULTIMAX
                NEXT
                SGI
                SUN
                SYMMETRY
                TITAN

To add more configurations, follow the examples of ifdefs for the Machine
dependent code sections.  The microsecond clock provided by the USC
package is used by ALOG to obtain the event timestamps.  Hence, all
timestamps appearing in the log files are expressed in microseconds.


ALOG TRACING INTERFACE
======================

The package comprises of the following macros/functions that are
provided via the include file "alog.h" and the library "libalog.a".


	ALOG_DEC:	

		no parameters 

		This macro is now redundant, but retained only for 
		backward compatibility.


	ALOG_SETUP(pid,flag):	

		pid  -  (integer) process id of callee
		flag -  (integer) either ALOG_WRAP or ALOG_TRUNCATE  

		This macros initializes the tracing area for a slave 
		process and MUST be called once before any event is 
		logged.  If the value of 'flag' is set to ALOG_WRAP, 
		then in the event of no more space for logging events 
		the system will only report the latest n events.  If 
		the 'flag' is set to ALOG_TRUNCATE the system will stop 
		logging events as soon as there is no more memory for 
		the events to be logged.


	ALOG_MASTER(pid,flag):

		pid  -  (integer) process id of the callee
		flag -  (integer) either 0 or 1  (see above)

		This macro has the same effect over its parameters as
		ALOG_SETUP with the difference that this macro should be 
		referenced by the master process only.


	ALOG_DEFINE(event,strdef,format):

		event  - (integer) id of event being defined
		strdef - (string) description of 'event'
		format - (string) control string in "printf" format to

		This macro expands to an event definition code.


	ALOG_LOG(pid,event,intdata,strdata):

		pid     - (integer) process id of callee
		event   - (integer) event id to be logged
		intdata - (integer) any integer data for this event
		strdata - (string) any string data (can be the null string)

		This macro provides the event logging service.


	ALOG_OUTPUT:
		no parameters  

		This macro dumps the events logged into a log file with
		the name "alogfile.pxx" where 'xx' is the logical PID
		of the callee process.  The log file is created in the
		current directory unless specified otherwise through
		the macro ALOG_SETDIR.


	ALOG_SETDIR(dir):

		dir - (string) directory where log file is created

		This macro sets the output directory for the log file.
		The default directory for the creation of the log file
		is the current directory of the process.  If used, then
		this macro MUST be invoked before ALOG_MASTER/ALOG_SETUP.


	ALOG_STATUS(status):

		status - (integer) either ALOG_ON or ALOG_OFF

		This macro controls the logging status of ALOG as follows.
		Setting 'status' to ALOG_ON enables logging until it is
		turned off.  Setting 'status' to ALOG_OFF disables logging 
		until it is turned on again.  Logging is enabled at the 
		outset by default.


	ALOG_ENABLE:

		no parameters

		This macro enables event logging; same as calling
		ALOG_STATUS(ALOG_ON).


	ALOG_DISABLE:

		no parameters

		This macro disables event logging; same as calling
		ALOG_STATUS(ALOG_OFF).

An application that uses the ALOG tracing facility must do the
following:

 1) Include the file "alog.h" at the beginning of the program.
 2) Link the application with the library file "libalog.a" and the
    machine-specific library mentioned below.


INCLUDE FILE
============

        alog.h


LIBRARY FILE
============

        libalog.a


ADDITIONAL MACHINE DEPENDENT LIBRARIES
======================================

The application will also have to be linked with the following machine
dependent libraries in order to enable the microsecond clock to function
properly:

                 MULTIMAX        -lpp
                 BALANCE         -lseq
                 SYMMETRY        -lseq

SHAR_EOF
if test -f 'alog.c'
then
	echo shar: over-writing existing file "'alog.c'"
fi
cat << \SHAR_EOF > 'alog.c'
#include <stdio.h>
#if !defined(SYMMETRY_PTX)
#include <strings.h>
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include "alog.h"

int xx_alog_status = 0x3;	/* Logging turned ON (default) */
int xx_alog_setup_called = 0;	/* setup  not yet called */
int xx_alog_output_called = 0;	/* output not yet called */
char xx_alog_outdir[MAX_DIRNAME_LEN+1] = "";
struct head_trace_buf *xx_buf_head = NULL;


VOID xx_write(head,pid,event,data1,data2)
struct head_trace_buf *head;
int pid,event,data1;
char *data2;
{
	int xx_i, xx_j, found;
	struct trace_buf *buf;

	if (xx_alog_status >> 1) {
		fprintf(stderr, "ALOG: Error: event logging requested by PID %d before doing ALOG setup\n", pid);
		return;
	}
	if (head == NULL) return;
	if (head->next_entry == head->max_size +99)
		return;
	found = 0;
	while(!found) {
	    if (head->next_entry >= head->max_size) {
		xx_i = 1;
		head->next_entry = 0;
		if (head->cbuf->next_buf == NULL)
			xx_i = xx_getbuf(head);
		else 
			head->cbuf = head->cbuf->next_buf;
		if ((!xx_i) && (head->trace_flag == ALOG_WRAP))
			head->cbuf = head->xx_list;
		else if (!xx_i) {
			head->next_entry = (head->max_size + 99);
			return;
		}
	    } /* endif */
	    buf = head->cbuf;
	    xx_i = head->next_entry;
	    head->next_entry++;
	    if (buf->ALOG_table[xx_i].event > (-1))
		found = 1;
	} /* end while */
	buf->ALOG_table[xx_i].id 	= pid;
	buf->ALOG_table[xx_i].task_id 	= 0;
	buf->ALOG_table[xx_i].event 	= event;
	buf->ALOG_table[xx_i].data_int	= data1;
	if (event == (-11)){
		buf->ALOG_table[xx_i].tstamp = atol(data2);
		buf->ALOG_table[xx_i].tind = 0;
		strcpy(buf->ALOG_table[xx_i].data_string," ");
		head->prev_time = 0;
	} else if (event < 0) {
		buf->ALOG_table[xx_i].tstamp = 0; 
		buf->ALOG_table[xx_i].tind = 0;
		head->prev_time = 0;
		strncpy(buf->ALOG_table[xx_i].data_string,data2,MAX_LOG_STRING_LEN);
		buf->ALOG_table[xx_i].data_string[MAX_LOG_STRING_LEN] = '\0';
	} else {
		buf->ALOG_table[xx_i].tstamp = (unsigned long) usc_clock(); 
		if (buf->ALOG_table[xx_i].tstamp < head->prev_time)
			head->ind_time++;
		buf->ALOG_table[xx_i].tind = head->ind_time;
		head->prev_time = buf->ALOG_table[xx_i].tstamp;
		strncpy(buf->ALOG_table[xx_i].data_string,data2,MAX_LOG_STRING_LEN);
		buf->ALOG_table[xx_i].data_string[MAX_LOG_STRING_LEN] = '\0';
	}
}


VOID xx_dump(head)
struct head_trace_buf *head;
{
	struct trace_buf *temp;
	FILE *fp;
	int cycle;
	register int xx_i;

	if (head == NULL) return;
	if (head->file_t == NULL) {
		fprintf(stderr,"ALOG: *** Trace file not written ***\n");
		return;
	}
	fp = head->file_t;
	temp = head->xx_list;
	for(xx_i = 0; xx_i< head->max_size; xx_i++)
		if (temp->ALOG_table[xx_i].event < 0)
			fprintf(fp,"%d %d %d %d %u %u %s\n",
				temp->ALOG_table[xx_i].event,
				temp->ALOG_table[xx_i].id,
				temp->ALOG_table[xx_i].task_id,
				temp->ALOG_table[xx_i].data_int,
				temp->ALOG_table[xx_i].tind,
				temp->ALOG_table[xx_i].tstamp,
				temp->ALOG_table[xx_i].data_string);

	if (head->trace_flag == ALOG_WRAP) {
		temp = head->cbuf;
		if (temp != NULL)
			xx_dump_aux(temp,fp,head->next_entry,head->max_size);
		cycle = 0;
		while((temp != NULL) && (!cycle)) {
			temp = temp->next_buf;
			if (temp == NULL)
				temp = head->xx_list;
			if (temp == head->cbuf)
				cycle = 1;
			else
				xx_dump_aux(temp,fp,0,head->max_size);
		};
		if (temp != NULL)
			xx_dump_aux(temp,fp,0,head->next_entry);
	} else {
		temp = head->xx_list;
		while(temp != NULL) {
			xx_dump_aux(temp,fp,0,head->max_size);
			temp = temp->next_buf;
		};
	};

	fclose(head->file_t);
}


VOID xx_dump_aux(buf,fp,xx_j,xx_k)
struct trace_buf *buf;
FILE *fp;
int xx_j, xx_k;
{
	register int xx_i;
	for(xx_i = xx_j; xx_i < xx_k; xx_i++)
		if ((buf->ALOG_table[xx_i].id != (-1)) && 
		    (buf->ALOG_table[xx_i].event >= 0)) 
			fprintf(fp,"%d %d %d %d %u %u %s\n",
				buf->ALOG_table[xx_i].event,
				buf->ALOG_table[xx_i].id,
				buf->ALOG_table[xx_i].task_id,
				buf->ALOG_table[xx_i].data_int,
				buf->ALOG_table[xx_i].tind,
				buf->ALOG_table[xx_i].tstamp,
				buf->ALOG_table[xx_i].data_string);
}


int xx_getbuf(head)
struct head_trace_buf *head;
{
	register int i;
	struct trace_buf *temp;

	if (head == NULL) return(0);
	temp = (struct trace_buf*) malloc(sizeof(struct trace_buf));
	if (temp == NULL)
	{
		fprintf(stderr,"alog out of memory\n");
		return(0);
	}
	else {	
		head->cbuf = temp;
		for(i=0;i<MAX_BUF_SIZE;i++)
			temp->ALOG_table[i].id = (-1);
		temp->next_buf = NULL;
		if (head->xx_list == NULL)
			head->xx_list = temp;
		else {
			temp = head->xx_list;
			while(temp->next_buf != NULL)
				temp = temp->next_buf;
			temp->next_buf = head->cbuf;
		};
		return(1);
	};
}


VOID xx_user(head,id)
struct head_trace_buf *head;
int id;
{
	char c[MAX_LOG_STRING_LEN], cd[100];

	if (head == NULL) return;
	if (!strcmp(xx_alog_outdir, ""))
		getwd(cd);
	else
		strcpy(cd, xx_alog_outdir);
	if (cd[strlen(cd)-1] != '/')
		strcat(cd, "/");
	strcat(cd,ALOG_LOGFILE);
	sprintf(c,"%d",id);
	strcat(cd,c);

	if ((head->file_t = fopen(cd,"w")) == NULL)
		fprintf(stderr,"ALOG: *** Trace file creation failure ***\n");
}


VOID xx_user1(head,id)
struct head_trace_buf *head;
int id;
{
	struct stat buf;
	char c[MAX_LOG_STRING_LEN], cd[100], x[100];

	if (head == NULL) return;
	if (!strcmp(xx_alog_outdir, ""))
		getwd(cd);
	else
		strcpy(cd, xx_alog_outdir);
	if (cd[strlen(cd)-1] != '/')
		strcat(cd, "/");
	strcat(cd,ALOG_LOGFILE);
	sprintf(c,"%d",id);
	strcat(cd,c);

	stat(cd,&buf);
	strcpy(cd,(char *) ctime(&buf.st_ctime));

	strcpy(x,"AL");
	strncat(x,cd+4,3);	
	strcat(x,"-");
	strncat(x,cd+8,2);	
	strcat(x,"-");
	strncat(x,cd+22,2);	

	strcpy(c,"            ");
	strncpy(c,x,12);
	xx_write(head,0,(-1),0,c);

	sprintf(x,"%lu", (unsigned long) usc_rollover_val());
	xx_write(head,0,(-11),0,x);
}


VOID xx_alog_setup(pid,flag)
int pid, flag;
{
	register int i;
	char c[MAX_LOG_STRING_LEN], cd[MAX_LOG_STRING_LEN];

	usc_init(); 
	xx_alog_status &= 0x1;    /* set initialized flag */
	xx_buf_head = (struct head_trace_buf *)
        		malloc(sizeof(struct head_trace_buf));
	if (xx_buf_head == NULL) {
        	fprintf(stderr,"ALOG: *** Trace buffer HEADER creation failure;\n");
        	fprintf(stderr,"      *** tracing is being disabled.\n");
		return;
	}
	xx_buf_head->next_entry = 0;
	xx_buf_head->max_size = MAX_BUF_SIZE;
	xx_buf_head->prev_time = 0;
	xx_buf_head->ind_time = 0;
	xx_buf_head->trace_flag = flag;
	xx_buf_head->xx_list = NULL;
	if (!(xx_getbuf(xx_buf_head)))
        	printf("ALOG: **** trace buffer creation failure ****\n");
	xx_user(xx_buf_head,(pid));
}
SHAR_EOF
if test -f 'alog.h'
then
	echo shar: over-writing existing file "'alog.h'"
fi
cat << \SHAR_EOF > 'alog.h'
#include <stdio.h>
#include "usc.h"

#define MAX_DIRNAME_LEN 	100
#define MAX_LOG_STRING_LEN 	12
#define MAX_BUF_SIZE       	50
#define ALOG_LOGFILE		"alogfile.p"

#define ALOG_TRUNCATE		0
#define ALOG_WRAP		1

#define ALOG_OFF		0
#define ALOG_ON			1

#define ALOG_EVENT_SYNC        -101
#define ALOG_EVENT_PAIR_A1     -102
#define ALOG_EVENT_PAIR_A2     -103
#define ALOG_EVENT_PAIR_B1     -104
 

struct head_trace_buf {
        int             next_entry;
        int             max_size;
        unsigned long   prev_time;
        unsigned long   ind_time;
        int             trace_flag;
        struct trace_buf *xx_list;
        struct trace_buf *cbuf;
        FILE            *file_t;
};

struct trace_buf {
        struct trace_buf *next_buf;
        struct trace_table {
                int     id;
                int     task_id;
                int     event;
                int     data_int;
                char    data_string[MAX_LOG_STRING_LEN+1];
                unsigned long     tind;
                unsigned long     tstamp;
        } ALOG_table[MAX_BUF_SIZE];
};

extern int xx_alog_status;
extern int xx_alog_setup_called;
extern int xx_alog_output_called;
extern char xx_alog_outdir[];
extern struct head_trace_buf *xx_buf_head;

/* 
 * Function declarations 
 */

VOID xx_write(), xx_dump(), xx_dump_aux(),
     xx_user(), xx_user1(), xx_alog_setup();
int  xx_getbuf();


#ifdef ALOG_TRACE

#define ALOG_DEC

#define ALOG_STATUS(status) \
	if ((status) == ALOG_ON) \
		xx_alog_status |= 0x1; \
	else \
		xx_alog_status &= ~0x1

#define ALOG_ENABLE ALOG_STATUS(ALOG_ON)

#define ALOG_DISABLE ALOG_STATUS(ALOG_OFF)

#define ALOG_SETDIR(dir) \
	{\
	strncpy(xx_alog_outdir,(dir),MAX_DIRNAME_LEN); \
	xx_alog_outdir[MAX_DIRNAME_LEN] = '\0'; \
	}


#define ALOG_SETUP(pid,flag) \
        {\
            if (xx_alog_status & 0x1 &&  !xx_alog_setup_called) \
	    {\
                xx_alog_setup_called = 1;\
                xx_alog_setup((pid),(flag));\
	    }\
        }

#define ALOG_MASTER(pid,flag) \
	{\
	    if (xx_alog_status & 0x1) \
	    {\
	        xx_alog_setup((pid),(flag)); \
	        xx_user1(xx_buf_head,(pid)); \
	    }\
	}

#define ALOG_DEFINE(event,edef,strdef) \
        {\
        if (xx_alog_status & 0x1) \
        {\
            xx_write(xx_buf_head,0,(-9),(event),(edef)); \
            xx_write(xx_buf_head,0,(-10),(event),(strdef)); \
        }\
        }

#define ALOG_LOG(pid,type,data1,data2) \
	{\
	if (xx_alog_status & 0x1) \
		xx_write(xx_buf_head,(pid),(type),(data1),(data2)); \
	}

#define ALOG_OUTPUT \
        {\
            if (xx_alog_status & 0x1  &&  !xx_alog_output_called) \
	    {\
                xx_alog_output_called = 1;\
                xx_dump(xx_buf_head);\
	    }\
        }

#else

#define ALOG_DEC 
#define ALOG_STATUS(a)
#define ALOG_ENABLE
#define ALOG_DISABLE
#define ALOG_SETDIR(a)
#define ALOG_SETUP(a,b)
#define ALOG_MASTER(a,b)
#define ALOG_DEFINE(a,b,c)
#define ALOG_LOG(a,b,c,d)
#define ALOG_OUTPUT

#endif

#if defined(IPSC860)      || defined(DELTA)        || \
    defined(TITAN)        || defined(SYMMETRY_PTX) || \
    defined(HP)
#define getwd(X)  getcwd(X)
#endif
SHAR_EOF
if test -f 'alogf.c'
then
	echo shar: over-writing existing file "'alogf.c'"
fi
cat << \SHAR_EOF > 'alogf.c'
#include "alog.h"


alogfm_(id,opt)			/* ALOG_MASTER */
int *id, *opt;
{
    ALOG_MASTER(*id,*opt);
}

alogfs_(id,opt)			/* ALOG_SETUP */
int *id, *opt;
{
    ALOG_SETUP(*id,*opt);
}

alogfd_(event,edef,fdef)	/* ALOG_DEFINE */
int *event;
char *edef, *fdef;
{
    ALOG_DEFINE(*event,edef,fdef);
}

alogfl_(id,etype,data,string)	/* ALOG_LOG */
int *id, *etype, *data;
char *string;
{
    /*
    fprintf(stderr,"alogl: id=%d, etype=%d, data=%d, string=%s\n",
	   *id,*etype,*data,string);
    fflush(stderr);
    */
    ALOG_LOG(*id,*etype,*data,string);
}

alogfo_()			/* ALOG_OUTPUT */
{
    ALOG_OUTPUT;
}

alogft_(status)			/* ALOG_STATUS */
int *status;
{
    ALOG_STATUS(*status);
}

alogfr_(dir)			/* ALOG_SETDIR */
char *dir;
{
    ALOG_SETDIR(dir);
}
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib
LIBS = $(MDEP_LIBS)
LIBLINK = ../bin

ALOG_HDR = alog.h
ALOG_SRC = alog.c alogf.c
ALOG_OBJ = alog.o alogf.o

default:	
	make libalog.a mergelogs

all:	
	make libalog.a mergelogs


CFLAGS = -D$(MACHINE)

alog.o:$(P)
	$(CC) $(CFLAGS) -I$(INCLUDEDIR) -c alog.c 

alogf.o:$(P)
	$(CC) $(CFLAGS) -I$(INCLUDEDIR) -c alogf.c 

$(ALOG_OBJ):  $(ALOG_SRC) $(ALOG_HDR)

libalog.a:  $(ALOG_OBJ)
	$(AR) $(LIBDIR)/libp4.a $?
	$(RANLIB) $(LIBDIR)/libp4.a
#	$(AR) $@ $?
#	$(RANLIB) $@

mergelogs: alog_evntdfs.h mergelogs.o
	$(CC) -o $(LIBLINK)/mergelogs mergelogs.o

adjlogs:  alog_evntdfs.h adjlogs.o
	$(CC) -o $(LIBLINK)/adjlogs adjlogs.o -lmp

realclean:	clean
	$(RM) -f libalog.a mergelogs adjlogs Makefile

SHAR_EOF
if test -f 'mergelogs.c'
then
	echo shar: over-writing existing file "'mergelogs.c'"
fi
cat << \SHAR_EOF > 'mergelogs.c'
/*
 * merge.c:  Program to take mutiple strand format logfiles, sort them,
 *           merge them, and prepend information to the final logfile.
 *
 * Algorithm:
 *     Copy all the negative events at the beginning of the logfiles to a
 *       temporary header file.
 *     Use the unix sort command to sort all the logfiles.
 *     Analyze the data, ignoring all events that are not > 0.
 *     Print to stdout the results of the analyzation, the header file,
 *       and the sorted log entries.
 *
 *	Modified Summer 1990 by James Arthur Kohl
 *		to combine count collection more efficiently.
 *
 *		Now it's REAL fast!
 */
#include <stdio.h>
#include <ctype.h>           /* for calling isdigit()             */
#include "alog_evntdfs.h"       /* Logfile definitions */

#define FALSE 0
#define TRUE !FALSE
#define MAX(x,y)	( (x) > (y) ? (x) : (y) )

#define C_DATA_LEN 50

#define DO_NEGATIVE 1
#define IGNORE_NEGATIVE 2

struct log_entry
{
	int proc_id;
	int task_id;
	int event;
	int i_data;
	char c_data[C_DATA_LEN];
	int time_slot;
	unsigned long time;
};

struct list_struct
{
	struct log_entry entry;
	struct list_struct *next;
} *log_table;

struct list_struct *log_ptr;

int entry_tot;

main(argc,argv)
int argc;
char *argv[];
{
	FILE *headerfp;

	char headerfile[255];

	int pid;

	if ( argc <= 1 ) 
		usage();

	pid = getpid();

	sprintf(headerfile,"/usr/tmp/log.header.%d",pid);

	if ( (headerfp=fopen(headerfile,"w")) == NULL )
	{
		fprintf(stderr,"merge: unable to create temp file %s.\n",headerfile);
		exit(0);
	}

	combine_files(argc,argv,headerfp);

	if ( (headerfp=fopen(headerfile,"r")) == NULL )
	{
		fprintf(stderr,"merge: unable to read temp file %s.\n",headerfile);
		exit(0);
	}

	fprintf(stderr,"Analyzing.\n");
	analyze(headerfp);

	fclose(headerfp);
	unlink(headerfile); 
	free(headerfile);
} /* main */


combine_files(argc,argv,headerfp)
int argc;
char *argv[];
FILE *headerfp;
{
	FILE *in;

	struct list_struct **files;
	struct list_struct *last;
	struct list_struct *ptr;

	struct log_entry *entry;

	unsigned long min_time;

	int min_event;
	int min_slot;
	int min;

	int all_eof;
	int eof_flag;
	int negflag;
	int num;
	int i;

	num = argc - 1;

	if ( (files=(struct list_struct **)
		malloc(num * sizeof(struct list_struct *))) == NULL )
	{
		fprintf(stderr,"merge: unable to allocate input data array.\n");
		exit(0);
	}

	for ( i=0 ; i < num ; i++ )
	{
		fprintf(stderr,"Reading %s\n",argv[i+1]);

		if ( (in=fopen(argv[i+1],"r")) == NULL )
		{
			fprintf(stderr,"merge: unable to read data file %s.\n",argv[i+1]);
			exit(0);
		}

		if ( (files[i]=(struct list_struct *)
			malloc(sizeof(struct list_struct))) == NULL )
		{
			fprintf(stderr,"merge: unable to allocate list struct.\n");
			exit(0);
		}

		ptr = files[i];

		last = NULL;

		do
		{
			read_logentry(in,&(ptr->entry),DO_NEGATIVE);

			if ( !(eof_flag=feof(in)) )
			{
				if ( (ptr->next=(struct list_struct *)
					malloc(sizeof(struct list_struct))) == NULL )
				{
					fprintf(stderr,"merge: unable to allocate list struct.\n");
					exit(0);
				}

				last = ptr;

				ptr = ptr->next;
			}

			else
			{
				if ( last != NULL )
					last->next = NULL;

				else
					files[i] = NULL;

				free(ptr);
				ptr = NULL;
			}
		}
		while ( !eof_flag );

		fclose(in);
	}

	fprintf(stderr,"Sorting.\n");

	all_eof = 0;

	do
	{
		negflag = 0;

		for ( i=0 ; i < num ; i++ )
		{
			if ( files[i] == NULL )
				continue;

			entry = &(files[i]->entry);

			if ( entry->event < 0 )
			{
				negflag++;

				fprintf(headerfp,"%d %d %d %d %d %lu %s\n",entry->event,
					entry->proc_id,entry->task_id,entry->i_data,
					entry->time_slot,entry->time,entry->c_data);

				files[i] = files[i]->next;

				if ( files[i] == NULL )
					all_eof++;
			}
		}
	}
	while ( negflag && all_eof < num );

	fclose(headerfp);

	log_table = log_ptr = NULL;

	entry_tot = 0;

	while ( all_eof < num )
	{
		min_event = 0;
		min_time = 0;
		min_slot = 0;

		min = -1;

		for ( i=0 ; i < num ; i++ )
		{
			if ( files[i] == NULL )
				continue;

			entry = &(files[i]->entry);

			if (
				(entry->time_slot == min_slot && entry->time < min_time)
				|| entry->time_slot < min_slot
				|| min == -1
				|| (entry->time_slot == min_slot && entry->time == min_time
					&& entry->event < min_event)
				)
			{
				min_event = entry->event;
				min_time = entry->time;
				min_slot = entry->time_slot;

				min = i;
			}
		}

		if ( log_ptr == NULL )
		{
			log_table = log_ptr = files[min];

			files[min] = files[min]->next;
		}

		else
		{
			log_ptr->next = files[min];

			files[min] = files[min]->next;

			log_ptr = log_ptr->next;
		}

		log_ptr->next = NULL;

		entry_tot++;

		if ( files[min] == NULL )
			all_eof++;
	}

	fprintf(stderr, "  %d total entries\n", entry_tot);
}


usage()
{
	fprintf(stderr,"mergelogs: mergelogs infile1 infile2 ...\n");
	fprintf(stderr,"  writes to stdout\n");

	exit(0);
}


/* analyze:  At this point, we have one large sorted file called :tname:.
 *           We want to prepend certain data to it.
 *           We also want to prepend the data from the file pointed to
 *             by :headerfp:.
 *           See balance:/usr/local/trace/README for info.
 *           
 */
analyze(headerfp)
FILE *headerfp;
{
	struct log_entry *entry;

	int proc_tot, task_tot, time_slot_tot, event_tot;
	int i;

	get_counts(&proc_tot,&task_tot,&event_tot,&time_slot_tot);

	fprintf(stderr, "  %d separate processors\n",  proc_tot);
	fprintf(stderr, "  %d separate tasks\n",  task_tot);
	fprintf(stderr, "  %d event types\n", event_tot);

	fprintf(stdout,"%d %d %d %d %d %lu\n",NUM_EVENTS,0,0,entry_tot,0,0L);
	fprintf(stdout,"%d %d %d %d %d %lu\n",NUM_PROCS,0,0,proc_tot,0,0L);
	fprintf(stdout,"%d %d %d %d %d %lu\n",NUM_TASKS,0,0,task_tot,0,0L);
	fprintf(stdout,"%d %d %d %d %d %lu\n",NUM_EVTYPES,0,0,event_tot,0,0L); 
	fprintf(stdout,"%d %d %d %d %d %lu\n",START_TIME,0,0,0,0,
		log_table->entry.time);
	fprintf(stdout,"%d %d %d %d %d %lu\n",END_TIME,0,0,0,0,
		log_ptr->entry.time);
	fprintf(stdout,"%d %d %d %d %d %lu\n",NUM_CYCLES,0,0,time_slot_tot,0,0L);

	dump_header(headerfp);

	log_ptr = log_table;

	while ( log_ptr != NULL )
	{
		entry = &(log_ptr->entry);

		fprintf(stdout,"%d %d %d %d %d %lu  %s\n",entry->event, 
			entry->proc_id,entry->task_id,entry->i_data,
			entry->time_slot,entry->time,entry->c_data);
	
		log_ptr = log_ptr->next;
	}
} /* analyze */  


dump_header(headerfp)
FILE *headerfp;
{
	char buf[512];
	int len;

	do
	{
		if ( len=fread(buf,sizeof(char),512,headerfp) )
			fwrite(buf,sizeof(char),len,stdout);
	}
	while ( !feof(headerfp) && len );
} /* dump_header */
    

read_logentry(fp,table,do_negs)
FILE *fp;
struct log_entry *table;
int do_negs;
{
	char buf[81];
	char *cp;

	int i;

	do
	{	
		fscanf(fp,"%d %d %d %d %d %lu",
			&(table->event),&(table->proc_id),&(table->task_id),
			&(table->i_data),&(table->time_slot),&(table->time));

		cp = table->c_data;

		i = 0;

		do
		{
			fscanf(fp,"%c",cp);
		}
		while ( *cp == ' ' || *cp == '\t' );

		i++;

		while ( *cp != '\n' && i < C_DATA_LEN )
		{
			fscanf(fp,"%c",++cp);

			i++;
		}

		*cp = '\0';

		/*
		if ( !feof(fp) && table->event == 0 )
			fprintf(stderr,"0 reading in.\n");
		*/
	}
	while( table->event < 0 && do_negs == IGNORE_NEGATIVE && !feof(fp) );
}


get_counts(ptot,ttot,etot,tstot)
int *ptot,*ttot,*etot,*tstot;
{
	struct list_struct *ptr;

	struct log_entry *entry;
	struct log_entry *last;

	int *p, *t, *e;

	int flag;
	int val;
	int i,j;

	if ( (p=(int *)malloc(sizeof(int) * entry_tot)) == NULL )
	{
		fprintf(stderr,"Not enough memory.\n");
		exit(0);
	}

	if ( (t=(int *)malloc(sizeof(int) * entry_tot)) == NULL )
	{
		fprintf(stderr,"Not enough memory.\n");
		exit(0);
	}

	if ( (e=(int *)malloc(sizeof(int) * entry_tot)) == NULL )
	{
		fprintf(stderr,"Not enough memory.\n");
		exit(0);
	}

	for ( i=0 ; i < entry_tot ; i++ )
	{
		p[i] = t[i] = e[i] = 0;
	}

	*ptot = *ttot = *etot = 0;

	*tstot = 1;

	ptr = log_table;

	last == NULL;

	while ( ptr != NULL )
	{
		entry = &(ptr->entry);

		flag = 0;

		val = entry->proc_id;

		for ( j=0 ; j < *ptot && !flag ; j++ )
		{
			if ( p[j] == val )
				flag++;
		}

		if ( !flag )
		{
			p[(*ptot)++] = val;
		}

		flag = 0;

		val = entry->task_id;

		for ( j=0 ; j < *ttot && !flag ; j++ )
		{
			if ( t[j] == val )
				flag++;
		}

		if ( !flag )
		{
			t[(*ttot)++] = val;
		}

		flag = 0;

		val = entry->event;

		for ( j=0 ; j < *etot && !flag ; j++ )
		{
			if ( e[j] == val )
				flag++;
		}

		if ( !flag )
		{
			e[(*etot)++] = val;
		}

                /* printf("tstot=%d *tstot=%d\n",tstot,*tstot); */
		if ( (last != NULL) && (entry->time_slot != last->time_slot) )
                        *tstot = *tstot + 1;
			/*  (*tstot)++;  */

		last = entry;

		ptr = ptr->next;
	}
        return(0);
}
SHAR_EOF
if test -f 'alog_evntdfs.h'
then
	echo shar: over-writing existing file "'alog_evntdfs.h'"
fi
cat << \SHAR_EOF > 'alog_evntdfs.h'
/****************************************************************************
These are the reserved event types for logfile header records.  Unspecified
fields are either 0 or (in the case of string data) null.

e_type	proc_id  task_id  int_data  cycle  timestamp  string_data

 -1                                                    creator and date
 -2                       # events
 -3                       # procs
 -4                       # tasks
 -5                       # event types
 -6                                         start_time
 -7                                         end_time
 -8                       # timer_cycles
 -9                       event_type                   description
-10                       event_type                   printf string

*************************************************************************/

#define SYSTEM_TYPE	-1
#define NUM_EVENTS	-2
#define NUM_PROCS	-3
#define NUM_TASKS	-4
#define NUM_EVTYPES	-5
#define START_TIME	-6
#define END_TIME	-7
#define NUM_CYCLES	-8
#define EVTYPE_DESC	-9
#define EPRINT_FORMAT  -10

#define ALOG_EVENT_SYNC        -101
#define ALOG_EVENT_PAIR_A1     -102
#define ALOG_EVENT_PAIR_A2     -103
#define ALOG_EVENT_PAIR_B1     -104
SHAR_EOF
if test -f 'adjlogs.c'
then
	echo shar: over-writing existing file "'adjlogs.c'"
fi
cat << \SHAR_EOF > 'adjlogs.c'
/*
 * adjustlogs.c:  Program to take multiple alog logfiles, extract events
 *                for synchronizing the clocks, and generating adjusted times.
 *                The files are replaced, allowing the use of other alog tools.
 *
 * -e n defines synchronization events
 * -a1 n -a2 m -b1 k define pair-exchange events used to compute clock offsets
 * (There are predefined values; these allow the user to define their own)
 *
 * Algorithm:
 *     Build a matrix of time events; solve it for the offset and skew for
 *     each clock.  For the first pass, this "matrix" will have just the 
 *     "synchronization" events.
 *
 * This is the formula:
 * Processor 0 has the standard clock.
 * At the end of each sync, the clock are re-synchronized.
 * Thus, the global time for processor p is
 *    Find the interval I in synctime that contains the local time
 *    The adjusted gtime is:
 *
 *            stime[0][I+1]-stime[0][I]
 *    gtime = ------------------------- (time - stime[p][I]) + stime[0][I]
 *            stime[p][I+1]-stime[p][I]
 * 
 * The current implementation uses a single interval.
 *
 * Just to keep things more interesting, the timer is really a 64 bit clock,
 * with the field "time_slot" containing the high bits.
 *
 */
#include <stdio.h>
#include <ctype.h>           /* for calling isdigit()             */
#include "alog_evntdfs.h"       /* Logfile definitions */

#define FALSE 0
#define TRUE !FALSE
#define MAX(x,y)	( (x) > (y) ? (x) : (y) )

#define C_DATA_LEN 50

#define DO_NEGATIVE 1
#define IGNORE_NEGATIVE 2

struct log_entry
{
	int proc_id;
	int task_id;
	int event;
	int i_data;
	char c_data[C_DATA_LEN];
	int time_slot;
	unsigned long time;
};

#define MAX_NSYNC 2
typedef struct {
    unsigned long *time;            /* time values that were recorded */
    } SyncTime;
SyncTime synctime[MAX_NSYNC];

/* For now, we just handle a set of timing events (np-1 of them)
   between processor i and i+1 (processor 0 participates in only
   1 event) */
typedef struct {
    unsigned long a1, b1, a2;         /* Times for the events */
    int           p0, p1;             /* processors that participated in
					 this time-exchange */
    } OffsetEvents;
OffsetEvents *offsetevents;
int noffsetevents = 0;

/* A local offset is used to compensate for the time_slot (upper 32 bits) */
/* NOT YET IMPLEMENTED */
long local_offset;

/* The global time is found by adding an offset and scaling by
   a fraction that is represented by numer[i]/denom[i] on the i'th
   processor */
unsigned long *numer;
unsigned long *denom;
long          *globaloffset;

/* mintime holds the mintime for ALL runs; this can be used to 
   offset the values */
unsigned long mintime;

/* These hold user-defined synchronization events */
#define MAX_USERETYPES 100
static int syncevent[MAX_USERETYPES];
static int syncep=0;

/* These hold the 3 event types used to adjust the individual offsets
   (if not present, the synchronization events are used to compute the
   offsets)
 */
static int a1event[MAX_USERETYPES],
           a2event[MAX_USERETYPES],
           b1event[MAX_USERETYPES];
static int a1p = 0, a2p = 0, b1p = 0;

static unsigned long lowmask = 0xFFFF;

void ComputeOffsets();
           
main(argc,argv)
int argc;
char *argv[];
{
	FILE *headerfp, *fd;
	int  np, i, nsync, nlsync;
	char headerfile[255];
	int pid;
        int firstfile;

	if ( argc <= 1 ) 
		usage( argv[0] );

	/* Look for user-defined events */
	for (i=1; i<argc; i++) {
	    if (strcmp(argv[i],"-e") == 0) 
		/* Test on MAX_USERTYPES */
		syncevent[syncep++] = atoi(argv[++i]);
	    else if (strcmp(argv[i],"-a1") == 0)
	    	a1event[a1p++] = atoi(argv[++i]);
	    else if (strcmp(argv[i],"-a2") == 0) 
	    	a2event[a2p++] = atoi(argv[++i]);
	    else if (strcmp(argv[i],"-b1") == 0) 
	    	b1event[b1p++] = atoi(argv[++i]);
	    else
		break;
	    }
	/* Figure out how many processors there are */
	np        = argc - i;
	firstfile = i;
	/* These could be allocated on demand */
	for (i=0; i<MAX_NSYNC; i++) {
	    synctime[i].time       = (unsigned long *)
		                malloc( np * sizeof(unsigned long) );
	    }
	    
	globaloffset = (long *) malloc( np * sizeof(long) );
	numer        = (unsigned long *) malloc( np * sizeof(unsigned long) );
	denom        = (unsigned long *) malloc( np * sizeof(unsigned long) );
	offsetevents = (OffsetEvents *) malloc( np * sizeof(OffsetEvents) );
	mintime      = (unsigned long)(~0);

	/* Loop through each file, looking for the synchronization events */
	for (i=0; i<np; i++) {
	    fd = fopen( argv[firstfile+i], "r" );
	    nsync = extract_timing( i, fd );
	    if (i > 0 && nsync != nlsync) {
		fprintf( stderr, "Found differing numbers of syncs\n" );
		exit(0);
		}
	    nlsync = nsync;
	    fclose( fd );
	    }
	/* If we didn't find enough events, we exit */
	if (nsync < 2) {
	    fprintf( stderr, 
		     "Not enough synchronization events to adjust logs\n" );
	    exit(0);
	    }

	/* Compute a "global clock" time */
	/* NOTE: if numer is changed, ComputeOffsets must be changed as well */
	for (i=0; i<np; i++) {
	    numer[i]        = (synctime[1].time[0] - synctime[0].time[0]);
	    denom[i]        = (synctime[1].time[i] - synctime[0].time[i]);
	    /* Using mintime here fails for some log files (since some of the
	       computed/scaled times can then be negative.  We have to pick
	       a value that makes the minimum COMPUTED time positive */
	    globaloffset[i] = synctime[0].time[i]; /*   - mintime; */
	    }
	fprintf( stderr, "Summary of clock transformations:\n" );
	if (noffsetevents == np - 1) {
	    /* Print out the initial globaloffsets */
	    fprintf( stderr, "Global offsets from sync events are:\n" );
	    for (i=0; i<np; i++) {
		fprintf( stderr, "%4d  %12ld\n", i, globaloffset[i] );
		}
	    }

	/* Use adjust events to compute a modified offset (if such events
	   are not present, the globaloffset values above will be used) */
	ComputeOffsets( np );

	/* Write a summary */
	for (i=0; i<np; i++) {
	    fprintf( stderr, "%4d  (t - %12ld) (%lu/%lu)\n", 
		     i, globaloffset[i], numer[i], denom[i] );
	    }
        
	/* Rewrite the log files using the clock adjustment */
	for (i=0; i<np; i++) {
	    pid = getpid();
	    sprintf( headerfile, "%s.new", argv[firstfile+i] );
/* 	    sprintf(headerfile,"log.header.%d",pid); */
	    if ( (headerfp=fopen(headerfile,"w")) == NULL ) {
		fprintf(stderr,"%s: unable to create temp file %s.\n",
			argv[0], headerfile );
		exit(0);
		}
	    fd = fopen( argv[firstfile+i], "r" );
	    if (!fd) {
		fprintf( stderr, "%s: Unable to open log file %s\n", 
			 argv[0], argv[firstfile+i] );
		exit(0);
		}
	    adjust_file( i, fd, headerfp, 0, nsync, argv[firstfile+i] );
	    fclose( fd );
	    fclose( headerfp );

	    /* move filename */
/* 	    unlink( argv[firstfile+i] );
	    link( headerfile, argv[firstfile+i] );
	    unlink( headerfile );  */
	    }
	
} /* main */

/*
   Extract timing data for the i'th log file 
 */
int extract_timing( i, fd )
int  i;
FILE *fd;
{
struct log_entry entry;
int    nsync = -1;

while (1) {
    read_logentry(fd,&entry,DO_NEGATIVE);
    if ( feof(fd) ) break;
    if (is_sync_event(entry.event)) {
	/* We do this so that we save the LAST sync event */
	if (nsync + 1 < MAX_NSYNC) nsync++;
	synctime[nsync].time[i] = entry.time;
	/* fprintf( stdout, "Event type %d at time %d on proc %d\n",
		 entry.event, entry.time, i ); */
	}
	/* For the offset events, the assumption is that each processor
	   (except for processor 0) is the ORIGINATOR of one offsetevent.
	   It MAY participate as the respondent (b1 event) for multiple
	   events, including having processor 0 respond to EVERYONE.
	   Finally, the (b1) processor has processor number SMALLER than
	   the (a1,a2) processor.  This makes the equations that need
	   to be solved for the offsets TRIANGULAR and easy.
	 */
    else if (is_a1_event(entry.event)) {
    	offsetevents[i].a1 = entry.time;
    	offsetevents[i].p0 = entry.i_data;
        }
    else if (is_a2_event(entry.event)) {
    	offsetevents[i].a2 = entry.time;
    	offsetevents[i].p0 = entry.i_data;
    	noffsetevents++;
        }
    else if (is_b1_event(entry.event)) {
    	if (entry.i_data < i) {
    	    fprintf( stderr,
	             "Improper offset event (originating processor %d\n", i );
    	    fprintf( stderr, "higher numbered than partner %d)\n", 
		     entry.i_data );
    	    exit(0);
    	    }
    	offsetevents[entry.i_data].b1 = entry.time;
    	offsetevents[entry.i_data].p1 = i;
        }
    else if (entry.event > 0) {
	if (mintime > entry.time) mintime = entry.time;
	}
    }
return nsync + 1;
}

adjust_file( p, fin, fout, leave_events, nsync, fname )
FILE *fin, *fout;
int  p, leave_events, nsync;
char *fname;
{
struct log_entry entry;
unsigned long GlobalTime(), gtime;
unsigned long lasttime;

/* lasttime is used to make sure that we don't mess up the log files without
   knowing it */
lasttime = 0; 
while (1) {
    read_logentry(fin,&entry,DO_NEGATIVE);
    if ( feof(fin) ) break;
    if (!leave_events && (entry.event == ALOG_EVENT_SYNC ||
			  entry.event == ALOG_EVENT_PAIR_A1 ||
			  entry.event == ALOG_EVENT_PAIR_A2 ||
			  entry.event == ALOG_EVENT_PAIR_B1)) continue;
    /* adjust to the global clock time */
    gtime = GlobalTime(entry.time,p,nsync);
    if (entry.event > 0) {
	if (gtime < lasttime) {
	    fprintf( stderr, "Error computing global times\n" );
	    fprintf( stderr, "Times are not properly sorted\n" );
	    fprintf( stderr, "Last time was %lu, current time is %lu\n", 
		     lasttime, gtime );
	    fprintf( stderr, "(original new time is %lu)\n", entry.time );
	    fprintf( stderr, "processing file %s\n", fname );
	    exit(0);
	    }
	else 
	    lasttime = gtime;
	}
    /* negative events are unchanged. */
    fprintf(fout,"%d %d %d %d %d %lu %s\n",entry.event,
	    entry.proc_id,entry.task_id,entry.i_data,
	    entry.time_slot, (entry.event >= 0) ? gtime : entry.time, 
	    entry.c_data);
    }
}

usage( a )
char *a;
{
	fprintf(stderr,"%s: %s infile1 infile2 ...\n",a,a);
	fprintf(stderr,"  updates files with synchronized clocks\n");

	exit(0);
}

read_logentry(fp,table,do_negs)
FILE *fp;
struct log_entry *table;
int do_negs;
{
	char buf[81];
	char *cp;

	int i;

	do
	{	
		fscanf(fp,"%d %d %d %d %d %lu",
			&(table->event),&(table->proc_id),&(table->task_id),
			&(table->i_data),&(table->time_slot),&(table->time));

		cp = table->c_data;

		i = 0;

		do
		{
			fscanf(fp,"%c",cp);
		}
		while ( *cp == ' ' || *cp == '\t' );

		i++;

		while ( *cp != '\n' && i < C_DATA_LEN )
		{
			fscanf(fp,"%c",++cp);

			i++;
		}

		*cp = '\0';

		/*
		if ( !feof(fp) && table->event == 0 )
			fprintf(stderr,"0 reading in.\n");
		*/
	}
	while( table->event < 0 && do_negs == IGNORE_NEGATIVE && !feof(fp) );
}

/* This routine allows use to define MANY sync events */
int is_sync_event( type )
int type;
{
int i;

if (type == ALOG_EVENT_SYNC) return 1;
for (i=0; i<syncep; i++) 
    if (type == syncevent[i]) return 1;
return 0;
}

int is_a1_event( type )
int type;
{
int i;

if (type == ALOG_EVENT_PAIR_A1) return 1;
for (i=0; i<a1p; i++) 
    if (type == a1event[i]) return 1;
return 0;
}

int is_a2_event( type )
int type;
{
int i;

if (type == ALOG_EVENT_PAIR_A2) return 1;
for (i=0; i<a2p; i++) 
    if (type == a2event[i]) return 1;
return 0;
}

int is_b1_event( type )
int type;
{
int i;

if (type == ALOG_EVENT_PAIR_B1) return 1;
for (i=0; i<b1p; i++) 
    if (type == b1event[i]) return 1;
return 0;
}

unsigned long GlobalTime( time, p, nsync )
unsigned long time;
int           p, nsync;
{
unsigned long gtime, stime1, stime2;
unsigned long frac;
unsigned long tdiff;
unsigned long ScaleLong();

/* Problem: since times are UNSIGNED, we have to be careful about how they
   are adjusted.  time - synctime may not be positive.  We make sure that
   all of the subexpressions are unsigned longs */
if (time >= globaloffset[p]) {
    tdiff = time - globaloffset[p];
    frac  = ScaleLong( numer[p], denom[p], tdiff );
    gtime = frac + globaloffset[0];
    }
else {
    tdiff = globaloffset[p] - time;
    frac  = ScaleLong( numer[p], denom[p], tdiff );
    if (frac > globaloffset[0]) printf( "Oops!\n" );
    gtime = globaloffset[0] - frac;
    }
return gtime;
}

/*
    This routine takes offset events and solves for the offsets.  The
    approach is:

    Let the global time be given by (local_time - offset)*scale ,
    with a different offset and scale on each processor.  Each processor
    originates exactly one communication event (except processor 0),
    generating an a1 and a2 event.  A corresponding number of b2 events
    are generated, but note that one processor may have more than 1 b2
    event (if using Dunnigan's synchronization, there will be np-1 b2 events
    on processor 0, and none anywhere else).

    These events are:

   pi   a1 (send to nbr)                        (recv) a2
   pj                     (recv) b1 (send back)

    We base the analysis on the assumption that in the GLOBAL time
    repreresentation, a2-a1 is twice the time to do a (send) and
    a (recv).  This is equivalent to assuming that global((a1+a2)/2) ==
    global(b1).  Then, with the unknowns the offsets (the scales
    are assumed known from the syncevent calculation), the matrix is

    1
    -s0 s1
       ....
       -sj ... si

    where si is the scale for the i'th processor (note s0 = 1).
    The right hand sides are (1/2)(a1(i)+a2(i)) *s(i) - b1(j)*s(j).
    Because of the triangular nature of the matrix, this reduces to

       o(i) = (a1(i)+a2(i))/2 - (s(j)/s(i)) * (b1(j)-o(j))

    Note that if s(i)==s(j) and b1 == (a1+a2)/2, this gives o(i)==o(j).
    
 */
void ComputeOffsets( np )
int np;
{
int i, j;
unsigned long d1, delta;
unsigned long ScaleLong();

/* If there aren't enough events, return */
if (noffsetevents != np - 1) {
    if (noffsetevents != 0) 
	fprintf( stderr, 
	   "Incorrect number of offset events to compute clock offsets\n" );
    else
	fprintf( stderr, "No clock offset events\n" );
    return;
    }

/* Take globaloffset[0] from sync */
for (i=1; i<np; i++) {
    /* o(i) = (a1(i)+a2(i))/2 - (s(j)/s(i)) * (b1(j)-o(j)) */
    j  = offsetevents[i].p1;
    
    /* Compute a1(i)+a2(i)/2.  Do this by adding half the difference;
       this insures that we avoid overflow */
    d1 = (offsetevents[i].a2 - offsetevents[i].a1)/2;
    d1 = offsetevents[i].a1 + d1;

    /* We form (b1-o(j))(s(j)/s(i)) by noting that
       s(j)/s(i) == denom(i)/denom(j) (since numer(i)==numer(j)) */
    delta = ScaleLong( denom[i], denom[j],
                       offsetevents[i].b1 - globaloffset[j] );

    globaloffset[i] = d1 - delta;
    }
}

#include <mp.h>
static MINT *prod, *qq, *rr;
static int  mpallocated = 0;

unsigned long ScaleLong( n, d, v )
unsigned long n, d, v;
{
char buf[40];
char *s;
MINT *nn, *dd, *vv;
unsigned long q, r;

if (!mpallocated) {
    prod = itom(0);
    if (!prod) {
	fprintf( stderr, "Could not allocate mp int\n" );
	exit(0);
	}
    qq   = itom(0);
    if (!qq) {
	fprintf( stderr, "Could not allocate mp int\n" );
	exit(0);
	}
    rr   = itom(0);
    if (!rr) {
	fprintf( stderr, "Could not allocate mp int\n" );
	exit(0);
	}
    mpallocated = 1;
    }

sprintf( buf, "%x", n );
nn = xtom(buf);
if (!nn) {
    fprintf( stderr, "Could not allocate mp int\n" );
    exit(0);
    }
sprintf( buf, "%x", v );
vv = xtom(buf);
if (!vv) {
    fprintf( stderr, "Could not allocate mp int\n" );
    exit(0);
    }
sprintf( buf, "%x", d );
dd = xtom(buf);
if (!dd) {
    fprintf( stderr, "Could not allocate mp int\n" );
    exit(0);
    }

mult(nn,vv,prod);
mdiv(prod,dd,qq,rr);
s = mtox(qq);
sscanf( s, "%x", &q );
free( s );
s = mtox(rr);
sscanf( s, "%x", &r );
free( s );

/* Free the locals */
mfree( nn );
mfree( dd );
mfree( vv );

return q;
}


/* Here is not-quite working code for multiple precision arithmetic */
#ifdef DO_MULTIPLE_ARITH

/* 
   This routine takes a value v and scales it by (n/d).  This 
   routine handles integer overflow by using the following formulas:
   Let h(u) = high 16 bits of u, and l(u) = low 16 bits of u.
   Then v *(n/d) = 

   (l(v)+h(v))*(l(u)+h(u))/d 
   = l(v)l(n)/d + (l(n)h(v)+l(v)h(n))/d + h(v)h(n)/d
   
   == a1/d + a2/d + a3/d

   In order to keep the values in-range, we define low(u)=l(u) and
   high(u) = h(u) >> 16.  Then this formula becomes (with high substituted
   for h):

   a1/d + (a2<<16)/d + (a3<<32)/d

   Now, when doing the integer division, we need to propagate the remainders.
   Let the result be r.  Then

   rd = a1 + (a2<<16) + (a3<<32)

   if a1 = k1 d + b1, (a2 << 16) = (k2 d + b2), and (a3 << 32) = (k3 d + b3),
   then
   
   r d = (k1 d + b1) + (k2 d + b2) + (k3 d + b3);
       = (k1 + k2 + k3) d + (b1 + b2 + b3)
       
   and so

   r   = (k1 + k2 + k3) + (b1 + b2 + b3) / d

   To compute (k2,b2) and (k3,b3), we do:

   (a2<<16)/d:

   a2 = p2 d + c2
   a2 << 16 = p2 d << 16 + c2 << 16
            = (p2 << 16) d + c2 << 16
   Let c2 << 16 = r2 d + s2, then (finally!)
   a2 << 16 = (p2 << 16 + r2) d + s2

   (a3 << 32)/d:

   a3 = p3 d + c3
   a3 << 32 = p3 d << 32 + c3 << 32
            = (p3 << 32) d + c3 << 32
   Computing c3 << 32 = r3 d + s3 is a challange, particularly
   since we need only the low 32 bits (the high 32 bits will be 0)
   We do this in stages as well:

   c3 << 32 = (c3 << 16) << 16; 
   (c3 << 16) = t3 d + u3
   (c3 << 32) = (t3 << 16)d + u3 << 16
              = (t3 << 16 + y3)d + z3,
	      == r3 d + s3
   where u3 << 16 = y3 d + z3

   Then
   a3 << 32 = (p3 << 32 + r3) d + s3
 */
void DivLong();

/* 
   ScaleDecomp - convert (a << p) = alpha d + beta, with beta < d

   This works by recursively:
   a = b d + r,
   a<<p = (b << p)d + (r<<p)
   then process r<<p to bd + r' etc until b == 0
 */
void ScaleDecomp( a, p, d, alpha, beta )
int           p;
unsigned long a, d, *alpha, *beta;
{
unsigned long b, r;
unsigned long Alpha, Beta;
int      p1;

Alpha = 0; Beta = 0;

b     = a / d;
r     = a % d;
Alpha = b << p;
/* We need to gingerly deal with r, since shifting it by much
   may make it too large, particularly if d is nearly 32 bits.  
   What we need is r << p = gamma d + delta, with r < d.  This
   is really the hard part. We can not assume that d is much
   smaller than 32 bits, so this is tricky. */

DivLong( r, d, (unsigned long)(1 << p), &b, &r );
Alpha += b;
*beta = r;
#ifdef FOO
while (p > 1 && r > 0) {
    p1    = p/2;
    r     = (r << p1);
    b     = r / d;
    r     = r % d;
    Alpha += b << (p-p1);
    p      = (p - p1);
    }
*alpha = Alpha;
*beta  = r << p;
#endif
}
#define LOWBITS(a) (unsigned long)((a)&lowmask)
#define HIGHBITS(a) (unsigned long)( ((a) >> 16 ) & lowmask )

#include <mp.h>
unsigned long ScaleLong( n, d, v )
unsigned long n, d, v;
{
#ifdef FOO
#define LOWBITS(a) (unsigned long)((a)&lowmask)
#define HIGHBITS(a) (unsigned long)( ((a) >> 16 ) & lowmask )
unsigned long a1, a21, a22, a3, k1, k21, k22, k3, b1, b21, b22, b3;

DivLong( n, d, v, &k1, &b1 );
return k1 + b1/d;

a1  = LOWBITS(v)*LOWBITS(n);
a21 = LOWBITS(v)*HIGHBITS(n);
a22 = LOWBITS(n)*HIGHBITS(v);
a3  = HIGHBITS(v)*HIGHBITS(n);

k1 = a1 / d;
b1 = a1 % d;

ScaleDecomp( a21, 16, d, &k21, &b21 );
ScaleDecomp( a22, 16, d, &k22, &b22 );
ScaleDecomp( a3,  32, d, &k3,  &b3 );

return (k1 + k21 + k22 + k3) + (b1 + b21 + b22 + b3) / d;
#else
char buf[40];
MINT *nn, *dd, *vv, *prod, *qq, *rr;
unsigned long q, r;

sprintf( buf, "%x", n );
nn = xtom(buf);
sprintf( buf, "%x", v );
vv = xtom(buf);
sprintf( buf, "%x", d );
dd = xtom(buf);
prod = itom(0);
qq   = itom(0);
rr   = itom(0);
mult(nn,vv,prod);
mdiv(prod,dd,qq,rr);
sscanf( mtox(qq), "%x", &q );
sscanf( mtox(rr), "%x", &r );
return q;
#endif
}

/*
  Represent nv = alpha d + beta
 */
void DivLong( n, d, v, alpha, beta )
unsigned long n, d, v;
unsigned long *alpha, *beta;
{
unsigned long a1, a21, a22, a3, k1, k21, k22, k3, b1, b21, b22, b3;

a1  = LOWBITS(v)*LOWBITS(n);
a21 = LOWBITS(v)*HIGHBITS(n);
a22 = LOWBITS(n)*HIGHBITS(v);
a3  = HIGHBITS(v)*HIGHBITS(n);

k1 = a1 / d;
b1 = a1 % d;

ScaleDecomp( a21, 16, d, &k21, &b21 );
ScaleDecomp( a22, 16, d, &k22, &b22 );
ScaleDecomp( a3,  32, d, &k3,  &b3 );

*alpha = k1 + k21 + k22 + k3;
*beta  = b1 + b21 + b22 + b3;
}

#endif
SHAR_EOF
cd ..
if test ! -d 'bin'
then
	mkdir 'bin'
fi
cd 'bin'
if test -f 'format'
then
	echo shar: over-writing existing file "'format'"
fi
cat << \SHAR_EOF > 'format'
#!/bin/csh
foreach fn (`ls p4*.c`)
    echo formatting $fn
    indent $fn temp -i4 -bl -di2 -npcs -nip -nce -npsl -nfc1 -cli0.5
    mv temp $fn
end
SHAR_EOF
chmod +x 'format'
if test -f 'killipc'
then
	echo shar: over-writing existing file "'killipc'"
fi
cat << \SHAR_EOF > 'killipc'
ipcrm `ipcs | awk '{if (($1 == "m") || ($1 == "s")) printf(" -%s %s ",$1,$2) }'`
SHAR_EOF
chmod +x 'killipc'
if test -f 'killp4'
then
	echo shar: over-writing existing file "'killp4'"
fi
cat << \SHAR_EOF > 'killp4'
PGFILE=$1
PGM=$2
echo $PGM | \
awk '{if (NR == 1)
      {
	  $APGM=$1
          printf("echo starting local kill \n")
          printf("/usr/local/p4/bin/killipc & \n")
          printf("/usr/local/p4/bin/kj %s & \n",$APGM)
      }
      else if ($1 != "local"  &&  substr($1,1,1) != "#") 
      {
          printf("echo starting kill on %s \n",$1)
          printf("%s %s /usr/local/p4/bin/killipc & \n","rsh",$1)
          printf("%s %s /usr/local/p4/bin/kj %s & \n","rsh",$1,$APGM)
      }
     }
     END { printf("echo all kills started \n") }  
     ' - $PGFILE  |  \
sh
SHAR_EOF
chmod +x 'killp4'
if test -f 'kj'
then
	echo shar: over-writing existing file "'kj'"
fi
cat << \SHAR_EOF > 'kj'
ps -auwx | grep $1  | \
grep -v grep | grep -v $$ | \
awk '{ print "kill -9 " $2 }' | sh
SHAR_EOF
chmod +x 'kj'
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIBS = ../lib/libp4.a $(MDEP_LIBS) 

default:$(P) 
	
all:$(P) 

realclean: clean
	$(RM) -f mergelogs serv_p4 serv_p4 killiserv_p4 iserv_p4 killiserv_p4 \
		 Makefile P4Server.Log.*
SHAR_EOF
if test -f 'makefuncs'
then
	echo shar: over-writing existing file "'makefuncs'"
fi
cat << \SHAR_EOF > 'makefuncs'
#!/bin/csh -f

#	Create the funcdecls.h file from the source in the directory

if ($#argv == 0) then
    set files="*.c"
else
    set files="$*"
endif

grep '^[A-Za-z].*)[^;]*$' $files | sed 's/^[^:]*://' | \
	grep -v '^[ 	]*static' | egrep -v '([^a-z_])?main\(' | \
	sed -e 's/ *$//' -e '/^[^ ]*$/s/^/int /' | sed 's/$/;/' | \
	sed 's/(.*)/()/' | \
	sort +1 | uniq
SHAR_EOF
chmod +x 'makefuncs'
if test -f 'makefuncs.tags'
then
	echo shar: over-writing existing file "'makefuncs.tags'"
fi
cat << \SHAR_EOF > 'makefuncs.tags'
awk \
    ' \
        /#define/ {}  \
        !/#define/ { print $3 " " $4 " " $5 " " $6 " " $7 " " $8 } \
    ' \
    < tags > temp1

sed -e "s/\/\^//" -e "s/(.*/();/" temp1 > temp2
SHAR_EOF
chmod +x 'makefuncs.tags'
if test -f 'start_servers'
then
	echo shar: over-writing existing file "'start_servers'"
fi
cat << \SHAR_EOF > 'start_servers'
#!/bin/sh
# script to start a set of programs on a set of machines (one each)
# takes as input a port number to use and a file containing machine - file
# pairs 
echo $1 | \
awk '{if (NR==1)
	{
         $APORT=$1
	}
        else
	{
         printf("rsh %s %s -d -p %s &\n",$1,$2,$APORT)
	 printf("echo starting %s on %s with port %s\n",$2,$1,$APORT)
	}
     }' - $2 | /bin/sh
SHAR_EOF
chmod +x 'start_servers'
if test -f 'kill_servers'
then
	echo shar: over-writing existing file "'kill_servers'"
fi
cat << \SHAR_EOF > 'kill_servers'
#!/bin/sh
# script to kill a set of programs on a set of machines (one each)
# takes as input a file containing machine - file pairs 
awk '{
         printf("rsh %s /usr/local/p4-1.1/bin/kj %s &\n",$1,$2)
	 printf("echo killing %s on %s\n",$2,$1)
     }' $1 | /bin/sh

SHAR_EOF
chmod +x 'kill_servers'
cd ..
if test ! -d 'contrib'
then
	mkdir 'contrib'
fi
cd 'contrib'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains assorted examples not part of the "official"
distribution and not automatically compiled when p4 is installed.  Current
programs include:

 lm_master, lm_slave - "lots of messages", a test program that stresses 
                       the message passing system 

 lm1_master, lm1_slave - "lots of messages", a test program that stresses 
                       the message passing system 

 simpglob  -  a simple example of the use of global operations in C

Use 

   make <program> to compile any of the above programs.


sockperf.c - a NON-p4 program for testing performance of sockets 
	     in various environments, i.e. when TCP_NODELAY is
	     or is not available, etc.
SHAR_EOF
if test -f 'lm.h'
then
	echo shar: over-writing existing file "'lm.h'"
fi
cat << \SHAR_EOF > 'lm.h'
#define CNTL 0
#define DATA 1
SHAR_EOF
if test -f 'lm.pg'
then
	echo shar: over-writing existing file "'lm.pg'"
fi
cat << \SHAR_EOF > 'lm.pg'
local 2
# sinkhole 3 /unf6/u/rbutler/p4/extra_examples/lm_slave
SHAR_EOF
if test -f 'lm1_master.c'
then
	echo shar: over-writing existing file "'lm1_master.c'"
fi
cat << \SHAR_EOF > 'lm1_master.c'
#include "p4.h"
#include "lm.h"

main(argc,argv)
int argc;
char **argv;
{
char buf[100];
int i, n;
int size;
int slv = 3;
int nbunches, bunchsize;
int nslaves;
int start, end;
int type;
char msg[200];
char *rcvd_msg;
int from,rcvd_msg_len;

    p4_initenv(&argc,argv);
    p4_create_procgroup();

    p4_dprintf("entering master user code\n");

    nslaves = p4_num_total_ids() - 1;
    p4_get_cluster_ids(&start, &end);

    p4_dprintfl(9,"got nslaves=%d start=%d end=%d\n",nslaves,start,end);

    printf("enter a number of bunches: ");
    scanf("%d",&nbunches);
    printf("enter size of a bunch: ");
    scanf("%d",&bunchsize);
	
    type = CNTL;
    sprintf(msg,"%d %d",nbunches,bunchsize);
    for (i=1; i <= nslaves; i++)
    {
	p4_dprintfl(9,"sending msg %s to %d size=%d\n",msg,i,sizeof(msg));
	p4_send(type, i, msg, sizeof(msg));
    }
    for (i=1; i <= nslaves; i++)
    {
        type = -1;
        from = -1;
	rcvd_msg= NULL;
	p4_recv(&type, &from, &rcvd_msg, &rcvd_msg_len);
	p4_dprintfl(9,"recvd msg from %d\n",from);
    }

    p4_dprintfl(9,"master entering waitforend\n");
    p4_wait_for_end();
    p4_dprintfl(9,"master past waitforend\n");
    p4_dprintf("exiting master user code\n");
}
SHAR_EOF
if test -f 'lm1_slave.c'
then
	echo shar: over-writing existing file "'lm1_slave.c'"
fi
cat << \SHAR_EOF > 'lm1_slave.c'
#include "p4.h"
#include "lm.h"
    
slave()	
{
    int myid;
    char buf[100];
    int size;
    int i,j,k,n;
    int nslaves;
    int start, end;
    char *msg;
    char loc_msg[200];
    int type;
    int done;
    int from;
    int nbunches,bunchsize;
    int value;
    int msg_cnt;
    int msgs_rcvd[P4_MAXPROCS];  /* cheated and used a p4 internal def */
    
    myid = p4_get_my_id();
    nslaves = p4_num_total_ids() - 1;
    p4_get_cluster_ids(&start, &end);
    /***
      p4_dprintf("Inside slave process %d: nslaves=%d start=%d end=%d\n",
      myid, nslaves, start, end);
      ***/
    
    for (i=0; i < P4_MAXPROCS; i++)
	msgs_rcvd[i] = 0;
    
    p4_dprintfl(9,"receiving...\n");
    type = CNTL;
    from = 0;
    msg = NULL;
    p4_recv(&type, &from, &msg, &size);
    p4_dprintfl(9,"rcvd from=%d type=%d msg=%s\n",from,type,msg);
    sscanf(msg,"%d %d",&nbunches,&bunchsize);
    p4_msg_free(msg);
    msgs_rcvd[from]++;
    
    type = DATA;
    for (j=0; j < nbunches; j++)
    {
	for (i=1; i <= nslaves; i++)
	{
	    for (k=0; k < bunchsize; k++)
	    {
		if (i != myid)
		{
		    sprintf(loc_msg,"%d",j);
		    p4_dprintfl(9,"sending %d %d to %d \n",j,k,i);
		    p4_send(DATA, i, loc_msg, sizeof(loc_msg));
		}
	    }
	}
    }

    done = bunchsize * nbunches * (nslaves - 1);
    msg_cnt = 0;
    while (msg_cnt < done)
    {
	/**  p4_dprintfl(0,"receiving \n");  **/
	type = DATA;
	from = -1;
	msg = NULL;
	p4_recv(&type, &from, &msg, &size);
	value = atoi(msg);
	msg_cnt++;
	p4_msg_free(msg);
	msgs_rcvd[from]++;
	p4_dprintfl(9,"rcvd from=%d type=%d value=%d\n",from,type,value);
    }
    p4_send(DATA, 0, loc_msg, sizeof(loc_msg));
    p4_dprintf("rcvd from: %d %d %d %d %d %d %d %d \n",
	    msgs_rcvd[0], msgs_rcvd[1], msgs_rcvd[2], msgs_rcvd[3],
	    msgs_rcvd[4], msgs_rcvd[5], msgs_rcvd[6], msgs_rcvd[7]);
	
    p4_dprintfl(0,"%d exiting\n", p4_get_my_id());
}
SHAR_EOF
if test -f 'lm_master.c'
then
	echo shar: over-writing existing file "'lm_master.c'"
fi
cat << \SHAR_EOF > 'lm_master.c'
#include "p4.h"
#include "lm.h"

main(argc,argv)
int argc;
char **argv;
{
char buf[100];
int i, n;
int size;
int slv = 3;
int nmsgs;
int nslaves;
int start, end;
int type;
char msg[200];
char *rcvd_msg;
int from,rcvd_msg_len;

    p4_initenv(&argc,argv);
    p4_create_procgroup();

    p4_dprintf("entering master user code\n");

    nslaves = p4_num_total_ids() - 1;
    p4_get_cluster_ids(&start, &end);

    p4_dprintfl(9,"got nslaves=%d start=%d end=%d\n",nslaves,start,end);

    printf("enter a number of messages:\n");
    scanf("%d",&nmsgs);
	
    type = CNTL;
    sprintf(msg,"%d",nmsgs);
    for (i=1; i <= nslaves; i++)
    {
	p4_dprintfl(9,"sending msg %s to %d size=%d\n",msg,i,sizeof(msg));
	p4_send(type, i, msg, sizeof(msg));
    }
    for (i=1; i <= nslaves; i++)
    {
        type = -1;
        from = -1;
	rcvd_msg = NULL;
	p4_recv(&type, &from, &rcvd_msg, &rcvd_msg_len);
	p4_dprintfl(9,"recvd msg from %d\n",from);
    }

    p4_dprintfl(9,"master entering waitforend\n");
    p4_wait_for_end();
    p4_dprintfl(9,"master past waitforend\n");
    p4_dprintf("exiting master user code\n");
}
SHAR_EOF
if test -f 'lm_slave.c'
then
	echo shar: over-writing existing file "'lm_slave.c'"
fi
cat << \SHAR_EOF > 'lm_slave.c'
#include "p4.h"
#include "lm.h"
    
slave()	
{
int myid;
char buf[100];
int size;
int i,j,n;
int nslaves;
int start, end;
char *msg;
char loc_msg[200];
int type;
int done;
int from;
int nmsgs;
int value;
int msg_cnt;
int msgs_rcvd[P4_MAXPROCS];  /* cheated and used a p4 internal def */

    myid = p4_get_my_id();
    nslaves = p4_num_total_ids() - 1;
    p4_get_cluster_ids(&start, &end);
    /***
    p4_dprintf("Inside slave process %d: nslaves=%d start=%d end=%d\n",
	     myid, nslaves, start, end);
    ***/

    for (i=0; i < P4_MAXPROCS; i++)
	msgs_rcvd[i] = 0;
    msg_cnt = 0;

    p4_dprintfl(9,"receiving...\n");
    type = CNTL;
    from = 0;
    msg = NULL;
    p4_recv(&type, &from, &msg, &size);
    p4_dprintfl(9,"rcvd from=%d type=%d msg=%s\n",from,type,msg);
    p4_msg_free(msg);
    msgs_rcvd[from]++;
    nmsgs = atoi(msg);

    type = DATA;
    for (j=1; j <= nmsgs; j++)
    {
	for (i=1; i <= nslaves; i++)
	{
	    if (i != myid)
	    {
		sprintf(loc_msg,"%d",j);
		p4_dprintfl(9,"sending %d to %d\n",j,i);
		p4_send(DATA, i, loc_msg, sizeof(loc_msg));
	    }
	}
    }

    done = nmsgs * (nslaves - 1);
    while (msg_cnt < done)
    {
	/**  p4_dprintfl(0,"receiving \n");  **/
	type = DATA;
	from = -1;
	msg = NULL;
	p4_recv(&type, &from, &msg, &size);
	value = atoi(msg);
	msg_cnt++;
	p4_msg_free(msg);
	msgs_rcvd[from]++;
	p4_dprintfl(9,"rcvd from=%d type=%d value=%d\n",from,type,value);
    }
    p4_send(DATA, 0, loc_msg, sizeof(loc_msg));
    p4_dprintf("rcvd from: %d %d %d %d %d %d %d %d \n",
	    msgs_rcvd[0], msgs_rcvd[1], msgs_rcvd[2], msgs_rcvd[3],
	    msgs_rcvd[4], msgs_rcvd[5], msgs_rcvd[6], msgs_rcvd[7]);
	
    p4_dprintfl(0,"%d exiting\n", p4_get_my_id());
}
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIBS = $(LIBDIR)/libp4.a $(MDEP_LIBS)

default:

all:
	make $(MFLAGS) lm_master lm1_master
	make $(MFLAGS) simpglob
	make $(MFLAGS) lm_slave lm1_slave

simpglob:$(P) simpglob.o  $(LIBDIR)/libp4.a
	$(CC) $(CFLAGS) -o simpglob simpglob.o  $(LIBS)

lm_master:$(P) lm_master.o lm_slave.o $(LIBDIR)/libp4.a
	$(CC) $(CFLAGS) -o lm_master lm_master.o lm_slave.o $(LIBS)

lm_slave:$(P) lm_slave.o $(LIBDIR)/libp4.a
	$(CC) $(CFLAGS) -o lm_slave $(LIBDIR)/p4_cmain.o lm_slave.o $(LIBS)

lm1_master:$(P) lm1_master.o lm1_slave.o $(LIBDIR)/libp4.a
	$(CC) $(CFLAGS) -o lm1_master lm1_master.o lm1_slave.o $(LIBS)

lm1_slave:$(P) lm1_slave.o $(LIBDIR)/libp4.a
	$(CC) $(CFLAGS) -o lm1_slave $(LIBDIR)/p4_cmain.o lm1_slave.o $(LIBS)

realclean: clean
	$(RM) -f lm_master lm_slave lm1_master lm1_slave \
	simpglob Makefile
SHAR_EOF
if test -f 'simpglob.c'
then
	echo shar: over-writing existing file "'simpglob.c'"
fi
cat << \SHAR_EOF > 'simpglob.c'
#include "p4.h"

#define ASIZE 10

main(argc,argv)
int argc;
char **argv;
{
    int i;

    p4_initenv(&argc,argv);
    if (p4_get_my_id() == 0)
        p4_create_procgroup();

    slave();

    p4_wait_for_end();
    p4_dprintf("exiting pgm\n");
}

slave()
{
int i, n;
double a[ASIZE];

    for (i=0; i < ASIZE; i++)
	a[i] = (double) i;

    p4_global_op(44,(char *) a,ASIZE,sizeof(double),p4_dbl_sum_op,P4DBL);

    for (i=0; i < ASIZE; i++)
	p4_dprintf("%4.1f\n",a[i]);
}
SHAR_EOF
if test -f 'sockperf.c'
then
	echo shar: over-writing existing file "'sockperf.c'"
fi
cat << \SHAR_EOF > 'sockperf.c'
#include <stdio.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <errno.h>
#include <sys/time.h>
#include <fcntl.h>
#include <netinet/tcp.h>
#include <sys/resource.h>

    
#define RECV_OK    0
#define RECV_EOF  -1
#define NON_RESERVED_PORT 5001

char *sys_errlist[];

/* Macros to convert from integer to net byte order and vice versa */
#define i_to_n(n)  (int) htonl( (u_long) n)
#define n_to_i(n)  (int) ntohl( (u_long) n)

char buf[1000];
struct sockaddr_in sin;

main(argc,argv)
int argc;
char *argv[];
{

    if (argc == 3)
    {
	printf("calling client\n");
	client(argv[1],atoi(argv[2]));
	printf("back from client\n");
    }
    else
    {
	printf("calling server\n");
	server();
	printf("back from server\n");
    }
}

server()
{
int i, rc, ssocket, sfd;
int optval = 1,optlen;
int flags;
int ntimes;

    setup_to_accept(5, NON_RESERVED_PORT, &ssocket);
    sfd = accept_connection(ssocket);

    flags = fcntl(sfd, F_GETFL, 0);
    if (flags < 0)
	error_check(flags,"getfl");
    flags |= O_NDELAY;
    fcntl(sfd, F_SETFL, flags);
    if (flags < 0)
	error_check(flags,"setfl");

    setsockopt(sfd,IPPROTO_TCP,TCP_NODELAY,(char *)&optval,sizeof(optval));
    rc = recv_msg(sfd, &ntimes, sizeof(int));
    printf("received ntimes=%d\n",ntimes);
    for (i=0; i < ntimes; i++)
    {
	while (!msgs_available(sfd))
	    ;
	rc = recv_msg(sfd, buf, 4);	
	rc = recv_msg(sfd, buf, 4);	
	send_msg(sfd, buf, 4);
	send_msg(sfd, buf, 4);
    }
    shutdown(sfd,0);
    shutdown(ssocket,0);
    unlink((char *)&sin);
}

client(server_host,ntimes)
char *server_host;
int ntimes;
{
int i, cfd,rc;
int optval = 1,optlen;
int flags, start_time, end_time;

    cfd = connect_to_server(server_host,NON_RESERVED_PORT);
    flags = fcntl(cfd, F_GETFL, 0);
    if (flags < 0)
	error_check(flags,"getfl");
    flags |= O_NDELAY;
    fcntl(cfd, F_SETFL, flags);
    if (flags < 0)
	error_check(flags,"setfl");

    setsockopt(cfd,IPPROTO_TCP,TCP_NODELAY,(char *)&optval,sizeof(optval));
    printf("sending ntimes=%d\n",ntimes);
    send_msg(cfd, &ntimes, sizeof(int));
    start_time = getclock();
    for (i=0; i < ntimes; i++)
    {
	send_msg(cfd, buf, 4);
	send_msg(cfd, buf, 4);
	while (!msgs_available(cfd))
	    ;
	rc = recv_msg(cfd, buf, 4);	
	rc = recv_msg(cfd, buf, 4);	
    }
    end_time = getclock();
    printf("time=%d\n",end_time-start_time);
}

msgs_available(fd)
int fd;
{
    int nfds = 0;
    fd_set read_fds;
    struct timeval tv;

    FD_ZERO(&read_fds);
    FD_SET(fd, &read_fds);
    tv.tv_sec = 0;
    tv.tv_usec = 0;
    nfds = select(24, &read_fds, 0, 0, &tv);
    if (nfds == -1)
    {
	error_check(nfds, "socket_send select");
    }
    return(nfds);
}

recv_msg(fd, buf, size)	
int fd;
char *buf;
int size;
{
    int recvd = 0;
    int n;

    while (recvd < size)
    {
	n = read(fd, buf + recvd, size - recvd);
        if (n < 0)
        {
	    continue;
            printf("%s :%d: %s\n","recvmsg ",n,sys_errlist[errno]);
	    return(0);
        }
	if (n > 0)
	    recvd += n;
    }
    return(recvd);
}


send_msg(fd, buf, size)	
int fd;
char *buf;
int size;
{
    int sent = 0;
    int sendsize,rc,n,nfds,done = 0;
    fd_set read_fds;
    struct timeval tv;

    while (sent < size)
    {
	if ((size - sent) > 4096)
	    sendsize = 4096;
	else
	    sendsize = size - sent;
        n = write(fd, buf + sent, sendsize);
        if (n < 0)
        {
            printf("%s :%d: %s\n","send_msg ",n,sys_errlist[errno]);
	    return(0);
        }
        if (n > 0)
            sent += n;
    }
    return(sent);
}

setup_to_accept(backlog, port, skt)	
int backlog;
int port;
int *skt;
{
int rc;

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = INADDR_ANY;
    sin.sin_port = htons(port);

    *skt = socket(AF_INET, SOCK_STREAM, 0);
    error_check(*skt,"net_setup_anon_listener socket");

    rc = bind(*skt, (struct sockaddr *)&sin ,sizeof(sin));
    error_check(rc,"net_setup_listener bind");

    rc = listen(*skt, backlog);
    error_check(rc,"net_setup_listener listen");
}

int accept_connection(skt)	
int skt;
{
struct sockaddr_in from;
int fromlen;
int skt2;
int gotit;

    fromlen = sizeof(from);
    gotit = 0;
    while (!gotit)
    {
	skt2 = accept(skt, (struct sockaddr *) &from, &fromlen);
	if (skt2 == -1)
	{
	    if (errno == EINTR)
		continue;
	    else
		error_check(skt2, "accept_connection accept");
	}
	else
	    gotit = 1;
    }

    return(skt2);
}

connect_to_server(hostname, port)	
char *hostname;
int port;
{
int s;
struct sockaddr_in listener;
struct hostent *hp;
int rc;

    hp = gethostbyname(hostname);
    if (hp == NULL)
    {
	printf("connect_to_server: gethostbyname %s: %s -- exiting\n",
		hostname, sys_errlist[errno]);
	exit(99);
    }

    bzero((void *)&listener, sizeof(listener));
    bcopy((void *)hp->h_addr, (void *)&listener.sin_addr, hp->h_length);
    listener.sin_family = hp->h_addrtype;
    listener.sin_port = htons(port);

    s = socket(AF_INET, SOCK_STREAM, 0);
    error_check(s, "net_connect_to_server socket");

    rc = connect(s,(struct sockaddr *) &listener, sizeof(listener));
    error_check(rc, "net_connect_to_server connect");

    return(s);
}

error_check(val, str)	
int val;
char *str;
{
    if (val < 0)
    {
	printf("%s :%d: %s\n", str, val, sys_errlist[errno]);
	exit(1);
    }
}


getclock()
{
    int i;
    struct timeval tp;
    struct timezone tzp;

    gettimeofday(&tp, &tzp);
    i = (int) tp.tv_sec;
    i *= 1000;
    i += (int) (tp.tv_usec / 1000);
    return(i);
}
SHAR_EOF
if test -f 'procgroup'
then
	echo shar: over-writing existing file "'procgroup'"
fi
cat << \SHAR_EOF > 'procgroup'
local 1
#sinkhole 2 /u4/rbutler/p4t03/contrib/temp
#sinkhole 2 /u4/rbutler/p4t03/contrib/temp
# campus.mcs.anl.gov 1 /sd23h/lusk/p4/contrib/systest/systest_slave
#backend 0
#+  4  /Net/anagram/anagram1/lusk/gamma/p4.node/contrib/lm_slave
SHAR_EOF
cd ..
if test ! -d 'contrib_f'
then
	mkdir 'contrib_f'
fi
cd 'contrib_f'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains some simple Fortran examples contributed by users.

norm - The master program constructs a matrix and distributes its 
       columns to the slave processes to compute the 1-norms of the 
       columns (the sums of the absolute values of the entries).

comm - A stress test for message passing.  Prompts for number of iterations, 
       and sends increasingly large messages (up to length 2^19) to the
       slaves and back in various configurations.

*.pg - The procgroup files necessary for running the above programs on
       various configurations of machines.
SHAR_EOF
if test -f 'comm_anagram.pg'
then
	echo shar: over-writing existing file "'comm_anagram.pg'"
fi
cat << \SHAR_EOF > 'comm_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/comm_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/comm_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/comm_slave

SHAR_EOF
if test -f 'comm_master.f'
then
	echo shar: over-writing existing file "'comm_master.f'"
fi
cat << \SHAR_EOF > 'comm_master.f'
      program master

      include 'p4f.h'

      integer i,slaves,type,from,start,done,time,retcde,recvlen
	  real a(600000)
      integer tagcnt, tagdat, tagend
      parameter (tagcnt = 10)
      parameter (tagdat = 20)
      parameter (tagend = 30)

      call p4init()
      call p4crpg()
      call p4setavlbuf(1,100)
      print *,'Master has started'

      n = 50
	  print *,'Enter interation count:'
	  read(*,*) n
      slaves = p4ntotids() - 1
      length = 8

      do 10 i = 1,slaves
         print *,'MASTER sending control message to ',i
      call p4sendr(tagcnt,i,n,length,retcde)
10    continue

C Initialize socket connections between the slaves
      call p4sendr(tagcnt,1,a,length,retcde)
      type = tagcnt
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)

      print *,'Number of slaves = ',slaves
      print *,' '
	  print *,'SEND to one slave only'         
	  do 55 i = 6,19
         length = 2**i  
		 time = 0
	     do 51 k = 1,n
      start = p4clock()
      call p4sendr(i,1,a,length,retcde)
      done = p4clock()
            time = done - start + time
51       continue
         time = time / n
         print *,'Total time to send message = ',time,
     &    '  bytes = ',length
C        print *,'Buffer pools------------------'
C        call p4avlbufs()
55    continue

      print *,' '
	  print *,'SENDR to one slave only'
	  do 60 i = 6,19
         length = 2**i  
		 time = 0
	     do 61 k = 1,n
      start = p4clock()
      call p4sendr(i,1,a,length,retcde)
      done = p4clock()
            time = done - start + time
61       continue
	     time = time / n
         print *,'Total time to send and acknowledge message = ',time,
     &    '  bytes = ',length
60    continue

      print *,' '
	  print *,'SEND and RECEIVE sent message from one slave only'
	  do 65 i = 6,19
         length = 2**i  
         time = 0
	     do 66 k = 1,n
      start = p4clock()
      call p4send(i,1,a,length,retcde)
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      done = p4clock()
            time = done - start + time
66       continue
         time = time / n
         print *,'Total time for send and receive = ',time,
     &	 '  bytes = ',length
65    continue
 
      print *,' '
	  print *,'SEND message and RECEIVE time from one slave'
	  do 67 i = 6,19
         length = 2**i  
		 time = 0
		 do 68 k = 1,n
      start = p4clock()
      call p4send(i,1,a,length,retcde)
      type = i
      from = -1
      call p4recv(type,from,done,4,recvlen,retcde)
            time = done - start + time
68       continue
		 time = time / n
         print *,'Total time for send and receive slave = ',time,
     &	 '  bytes = ',length
67    continue

      print *,' '
	  print *,'SEND message around a ring of slaves'
	  do 70 i = 6,19
         length = 2**i  
		 time = 0
		 do 72 k = 1,n
      start = p4clock()
      call p4send(i,1,a,length,retcde)
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      done = p4clock()
            time = done - start + time
72       continue
         time = time / n
         print *,'Total time for broadcast = ',time,'  bytes = ',length
70    continue

      print *,' '
	  print *,'SENDR message to each slave sequentially'
	  do 80 i = 6,19
         length = 2**i  
		 time = 0
		 do 81 k = 1,n
      start = p4clock()
	        do 82 j = 1,slaves
      call p4sendr(i,j,a,length,retcde)
82          continue
      done = p4clock()
         time = done - start + time
81       continue
		 time = time / n
         print *,'Total time for broadcast = ',time,'  bytes = ',length
80    continue

      length = 0
      do 90 i = 1,slaves
	  print *,'Master sending end message to ',i
      call p4sendr(tagend,i,a,length,retcde)
90    continue

999     call p4cleanup()
      print *,'Master exiting and has cleaned up'
      end

SHAR_EOF
if test -f 'comm_slave.f'
then
	echo shar: over-writing existing file "'comm_slave.f'"
fi
cat << \SHAR_EOF > 'comm_slave.f'
      subroutine fslave()

      include 'p4f.h'

      real a(600000)
      integer type, from, next, procid, length, numsl, retcde, recvlen
      integer tagcnt, tagdat, tagend, done
      parameter (tagcnt = 10)
      parameter (tagdat = 20)
      parameter (tagend = 30)

      numsl = p4ntotids() - 1
      procid = p4myid()
      print *,'slave ',procid,' has started'
      call p4flush()
      if (procid .eq. numsl) then
	     next = 0
      else
	     next = procid + 1
      endif
      length = 8
      type = tagcnt
      from = -1
      call p4recv(type,from,n,length,recvlen,retcde)
      print *,'SLAVE ',procid,' received from = ',from,' type = ',type
      call p4flush()

c Initialize socket connections between the slaves 
      type = tagcnt
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      call p4sendr(tagcnt,next,a,recvlen,retcde)

      length = 600000
c
	  if (procid .eq. 1) then
	     do 10 i = 6,19
		    do 11 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
11          continue
10       continue
	     do 22 i = 6,19
			do 21 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
21          continue
22       continue
	     do 23 i = 6,19
			do 24 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      call p4send(i,0,a,recvlen,retcde)
24          continue
23       continue
	     do 27 i = 6,19
			do 28 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      done = p4clock()
      call p4send(i,0,done,4,retcde)
28          continue
27       continue
      endif
c
	  do 30 i = 6,19
		 do 31 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
      call p4send(i,next,a,recvlen,retcde)
31       continue
30    continue
c
	  do 40 i = 6,19
		 do 41 k = 1,n
      type = i
      from = -1
      call p4recv(type,from,a,length,recvlen,retcde)
41       continue
40    continue
c
      length = 0
      type = tagend
      from = 0
      call p4recv(type,from,a,length,recvlen,retcde)
999   print *,'Slave ',procid,' has been cleaned up'
      call p4flush()
      end
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIB = $(LIBDIR)/libp4.a
FORTLIBDIR = $(P4_HOME_DIR)/lib_f
FORTLIB = $(FORTLIBDIR)/libp4_f.a

MAIN = $(FORTLIBDIR)/p4_fmain.o

LIBS =  $(FORTLIB) $(LIB) $(MDEP_LIBS)

default:$(P)

all:$(P)
	make $(MFLAGS) norm_master norm_slave
	make $(MFLAGS) comm_master comm_slave

norm_master: norm_master.o norm_slave.o $(FORTLIB)  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o norm_master norm_master.o norm_slave.o $(LIBS)
	
norm_slave: norm_slave.o  $(FORTLIB)  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o norm_slave $(MAIN) norm_slave.o $(LIBS)

comm_master: comm_master.o comm_slave.o $(LIB)
	$(FC) $(MDEP_FFLAGS) -o comm_master comm_master.o comm_slave.o $(LIBS)
	
comm_slave: comm_slave.o  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o comm_slave $(MAIN) comm_slave.o $(LIBS)

realclean: clean
	$(RM) -f norm_master norm_slave comm_master comm_slave \
	Makefile p4f.h
SHAR_EOF
if test -f 'norm_anagram.pg'
then
	echo shar: over-writing existing file "'norm_anagram.pg'"
fi
cat << \SHAR_EOF > 'norm_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/norm_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/norm_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/norm_slave
#anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/norm_slave
SHAR_EOF
if test -f 'norm_master.f'
then
	echo shar: over-writing existing file "'norm_master.f'"
fi
cat << \SHAR_EOF > 'norm_master.f'
      program m1

      include 'p4f.h'

*-----
*  file:-  norm_master.f
*  related files:-  
*    norm_slave.f         (slave program)
*    norm_<hostname>.pg   (process group file)
*
*  Test program for new p4 library.
*    Generate matrix,
*    distribute columns to slaves,
*    get sum norm of columns, verify and display result.
*
*  features:-
*    max vector length <= `NMAX'.  If you increase this value,
*            be sure to increase the same value in slave program, too.
*    4 data byte per integer (controlled by `LENINT')
*    4 data byte per item    (real; controlled by `LENREAL')
*  
*  Debug code disabled by `c$$$'.
*  status:- plain vanilla, no error control
*  Volker Kurz, ANL & U Frankfurt, 04-Oct-91
*-----
*     .. p4 routines ..
      external   p4init, p4crpg, p4ntotids, p4send, p4recv, p4cleanup,
     $     p4clock
*
*     .. constants ..
      integer    NMAX,      LENINT,   LENREAL,   OFF
      parameter (NMAX=200 , LENINT=4, LENREAL=4, OFF=-1)
      integer    TAGCNT,    TAGDAT,    TAGNEW,    TAGEND
      parameter (TAGCNT=10, TAGDAT=20, TAGNEW=30, TAGEND=40)
*
*     .. variables and arrays ..
      integer    nslaves, i, k, mnsl, time1, time2,
     $     iretcd, ireclen, ip, msglen, itype
      real       a(NMAX, NMAX), r(NMAX), rnorm
      logical    errors
*-----
*
      write (*,*) 'setting up parallel environment...'
*
      call p4init ()
      call p4crpg ()
      nslaves = p4ntotids() - 1
*
      write (*,*) 'initializing matrix...'
*
      do 1 k=1,NMAX
         a(1,k) = float (k+1)
         do 11 i=2,NMAX
            a(i,k) = 1.0
 11      continue
 1    continue
*-----------------------------------------------------------------------
*  Beginning of main loop
*  ~~~~~~~~~~~~~~~~~~~~~~
 2    continue
*
*  Ask for number of rows and columns.
*  Provide a condition to end the program.
*  Watch out, if they want to fool us by entering too big numbers.
*  Calculate message length for slave data.
*  Broadcast vector length to slaves.
*
      write (*,9993) NMAX
      read (*,*) m
      if (m.lt.1) then
         goto 7
      elseif (m.gt.NMAX) then
         m = NMAX
         write (*,*) 'value truncated to m = ',m
      endif
      write (*,9995) NMAX
      read (*,*) n
      if (n.lt.1) then
         goto 7
      elseif (n.gt.NMAX) then
         n = NMAX
         write (*,*) 'value truncated to n = ',n
      endif
*
      msglen = m * LENREAL
      time1 = p4clock()
*
      do 3 i=1,nslaves
         call p4send (TAGCNT, i, m, LENINT, iretcd)
 3    continue
*
*  Compute norms of columns
*  ~~~~~~~~~~~~~~~~~~~~~~~~
*  Process columns by strips of length 'nslaves'.
*  'mnsl' ensures that only 'n' columns are processed.
*  1st sweep dials out columns,
*  2nd sweep collects results.
*  
      do 4 i=1,n,nslaves
*        .. distribute columns to slaves ..
         mnsl = min ( n-i+1, nslaves ) 
         do 41 k=1,mnsl
*           .. send column i+k-1 to slave k ..
            call p4send (TAGDAT, k, a(1,i+k-1), msglen, iretcd)
 41      continue
         do 42 k=1,mnsl
*           .. receive norm from slave ..
            itype = TAGDAT
            ip = OFF
            call p4recv (itype, ip, rnorm, LENREAL, ireclen, iretcd)
*           .. result is for column i+ip-1 ..
            r(i+ip-1) = rnorm
 42      continue
 4    continue 
      time2 = p4clock()
*
*  Check results and report any errors.  
*  Norm of column k is:  #rows + k
*  Note: There should be no rounding errors!
*
      errors = .FALSE.
      do 5 k=1,n
         if (r(k).ne.m+k) then
            write (*,9999) k, m+k, r(k)
            errors = .TRUE.
         endif
c$$$         write (*,*) 'norm of col', k,' is:', r(k)
 5    continue
      if (errors) then
         write (*,*) 'mission completed, error(s) detected.'
      else
         write (*,*) 'mission completed, no errors detected.'
      endif
      write (*,9997) float (time2-time1) / 1000.0
*
*  Tell slaves to get ready for a new session.
*  Data is ignored, when slave detects data type TAGNEW.
*
      do 6 i=1,nslaves
         call p4send (TAGNEW, i, m, LENINT, iretcd)
 6    continue
*
*  End of main loop
*  ~~~~~~~~~~~~~~~~
      goto 2
 7    continue
*------------------------------------------------------------------------------
*
*  Terminate parallel session
*  ~~~~~~~~~~~~~~~~~~~~~~~~~~
*  Broadcast TAGEND signal to all slaves.
*  A slave will terminate, when TAGEND is detected,
*  the contents of the message will be ignored.
*
      do 8 i=1,nslaves
         call p4send (TAGEND, i, n, LENINT, iretcd)
 8    continue
      call p4cleanup
*-----
      stop
 9993 format(1x,'-----'/
     $     ' input number of rows (1 <= m <=',i7,')'/
     $     ' (any value <= 0 terminates program)')
 9995 format(1x,
     $     ' input number of columns (1 <= n <=',i7,')'/
     $     ' (any value <= 0 terminates program)')
 9997 format(1x,'time :',f12.3,' s')
 9999 format(1x,'Error in column ',i7/
     $     '   norm should be       :',i10/
     $     '   but computed value is:',e14.4)
      end
SHAR_EOF
if test -f 'norm_slave.f'
then
	echo shar: over-writing existing file "'norm_slave.f'"
fi
cat << \SHAR_EOF > 'norm_slave.f'
      subroutine fslave ()

      include 'p4f.h'

*-----
*  file:- norm_slave.f
*
*  Test program for new p4 fortran library.
*    Receive vector from master,
*    compute sum norm,
*    send result back to master.
*
*  features:-
*    max vector length <= 200   (controlled by `NMAX')
*    4 data byte per integer    (controlled by `LENINT')
*    4 data byte per item       (real; controlled by `LENREAL')
*
*  Debug code disabled by `c$$$'.
*  status:- plain vanilla, no error control
*  Volker Kurz, ANL & U Frankfurt, 03-Oct-91
*-----
*     .. p4 routines ..
      external   p4myid, p4recv, p4send
*
*     .. constants ..
      integer    NMAX,      LENINT,   LENREAL,   OFF,    MASTER
      parameter (NMAX=200 , LENINT=4, LENREAL=4, OFF=-1, MASTER=0)
      integer    TAGCNT,    TAGDAT,    TAGNEW,    TAGEND
      parameter (TAGCNT=10, TAGDAT=20, TAGNEW=30, TAGEND=40)
      real       ZERO
      parameter (ZERO=0.0)
*
*     .. variables and arrays ..
      integer    myid, n, i, itype, iretcd, ireclen, iproc, msglen
      real       v(NMAX), rnorm
*-----
      myid = p4myid()
c$$$      write (*,*) 'slave ', myid, ' ready.'
c$$$      call flush ()
*
*  Outer loop for different matrices
*  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 1    continue
*
*     .. receive vector length ..
      itype = OFF
      iproc = MASTER
      call p4recv (itype, iproc, n, LENINT, ireclen, iretcd)
c$$$      write (*,*) 'retcod from recv = ', iretcd, ' id = ', myid
c$$$      write (*,*) 'ireclen from recv = ', ireclen, ' id = ', myid
c$$$      call flush
      if (itype.eq.TAGEND) then
         write (*,*) 'slave', myid, ' ended normally'
         return
      elseif (itype.ne.TAGCNT) then
         write (*,*)
     $        'slave', myid, ' received unexpected data type:', itype
         return
      endif

*  Main loop for calculation
*  ~~~~~~~~~~~~~~~~~~~~~~~~~
*  Receive one vector at a time from 'MASTER'.  
*  Decide upon data type 'itype', whether norm shall be computed
*  or program terminated.
*  Watch out for unknown data type.
*
 2    continue
*        .. receive vector ..
         itype = OFF
         iproc = MASTER
         msglen = n*LENREAL
c$$$         write (*,*) 'recving vec  id = ', myid
c$$$         call flush
         call p4recv (itype, iproc, v, msglen, ireclen, iretcd)
c$$$         write (*,*) 'recvd vec  id = ', myid
c$$$         call flush
         if (itype.eq.TAGDAT) then
*           .. compute norm ..
            rnorm = ZERO
            do 22 i=1,n
               rnorm = rnorm + abs (v(i))
 22         continue 
*           .. send result to host ..
            call p4send (TAGDAT, MASTER, rnorm, LENREAL, iretcd)
         elseif (itype.eq.TAGNEW) then
*           .. end of outer loop for matrices ..
            goto 1
         elseif (itype.eq.TAGEND) then
            write (*,*) 'slave', myid, ' ended normally'
            return
         else
            write (*,*)
     $           'slave', myid, ' received unexpected data type:', itype
            return
         endif
      goto 2
*-----
      end
SHAR_EOF
if test -f 'procgroup'
then
	echo shar: over-writing existing file "'procgroup'"
fi
cat << \SHAR_EOF > 'procgroup'
local 1
#hurley.tcg.anl.gov 2 /usr8/lusk/tmp/comm_slave
#fanbelt 1 /home/rbutler/8h/p4t03/contrib_f/temps
#anagram 1 /Net/anagram/anagram8/lusk/p4/contrib_f/sr_slave
SHAR_EOF
cd ..
if test ! -d 'doc'
then
	mkdir 'doc'
fi
cd 'doc'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
The principal files in this directory are:


 p4.1          - A man page

 p4f.1         - A man page for the fortran interface to p4

 p4.tex        - The latex version of the manual

 latexinfo.sty - style file for latex-ing the manual

 p4.info       - The `info' version of this manual, ready to be put
                     into the directory where `info' files are kept 
		     at your site.  Check the value of your `emacs' 
		     variable INFO-DIRECTORY.

 p4.txt        - plain ascii text of the manual, in case nothing 
		     else works.

SHAR_EOF
if test -f 'p4.tex'
then
	echo shar: over-writing existing file "'p4.tex'"
fi
cat << \SHAR_EOF > 'p4.tex'
% \documentstyle[11pt,titlepage,latexinfo,twoside]{article}
\documentstyle[11pt,titlepage,latexinfo]{article}
\textwidth=6.0in
\textheight=9.0in
\hoffset=-0.5in
\voffset=-1.0in
\parskip=.1in
\pagestyle{empty}

\begin{document}
\bibliographystyle{plain}

\newindex{cp}
\newindex{fn}


{\advance\leftskip 4.0in
\noindent
Distribution Category: \\
Mathematics and \\ 
\null\nobreak\hspace{.25in} Computer Science (UC-405)
\par
}

\begin{center}

\vspace{.2in}
{\large
ARGONNE NATIONAL LABORATORY\\
}
9700 South Cass Avenue \\
Argonne, IL 60439-4801
\end{center}

\vspace{.06in}
\begin{center}
{\large\bf
-----------------\\
ANL-92/17 \\
-----------------\\
}
\end{center}
\vskip .75 in
\begin{center}
{\Large\bf
User's Guide to the p4 Programming System}
\end{center}
\vskip .1 in
\begin{center}
by
\end{center}
\vskip .1in
\begin{center}
{\large \em Ralph Butler* and Ewing Lusk}
\vskip .5 in
Mathematics and Computer Science Division

\vskip 1.6 in
October 1992
\end{center}
\vfill
\begin{quote}
{\small This work was supported in part by
the Applied Mathematical Sciences subprogram of the Office of Energy
Research under
Contract W-31-109-Eng-38.}

\smallskip

\noindent
*Also of The University of North Florida, Department of Computer Science, 
Jacksonville, Florida

\end{quote}

\clearpage
\pagestyle{headings}
\pagenumbering{roman}
\tableofcontents
\clearpage
\pagenumbering{arabic}

\setfilename{p4.info}

\node Top, Introduction, (dir) , (dir)


\begin{menu}
* Introduction::        
* Structure of the Distribution Directory::
* Installing p4::       
* Getting Started::     
* Specifying Processes in the Procgroup File::
* Developing a Simple p4 Program::
* Command-Line Arguments::
* The p4 Function Library::
* p4 Functions for Managing Processes and Clusters::
* Functions for Message Passing::
* Functions for Shared Memory::
* Functions for Timing p4 Programs::
* Functions for Debugging p4 Programs::
* Miscellaneous Functions::
* Fortran Interface::   
* Faster Startup with the Secure Server::
* Utilities for Managing a p4 Session::
* Creating Logfiles for Upshot::
* Machine-Specific Notes::
* Some Common Problems and their Solutions::
* Concept Index::       
* Function Index::      
\end{menu}

\node Introduction,Structure of the Distribution Directory,	,Top

\section{Introduction}

P4 is a library of macros and subroutines developed at Argonne National
Laboratory for programming a variety of parallel machines.  Its
predecessor was the m4-based ``Argonne macros'' system described in the
Holt, Rinehart, and Winston book {\em Portable Programs for Parallel
Processors}, by Lusk, Overbeek, et al., from which p4 takes its
name\cite{lusk-overbeek:p4-book}.  The current p4 system maintains the
same basic computational models described there (monitors for the
shared-memory model, message-passing for the distributed-memory model,
and support for combining the two models) while significantly increasing
ease and flexibility of use. \xref{Getting Started} for a simple example.

P4 is intended to be portable, simple to install and use, and efficient.  It
can be used to program networks of workstations, advanced parallel
supercomputers like the Intel Touchstone Delta and the Alliant Campus
HiPPI-based system, and single shared-memory multiprocessors.  It has
currently been installed on the following list of machines: Sequent Symmetry
(Dynix and PTX), Encore Multimax, Alliant FX/8, FX/800, and FX/2800, Cray
X/MP, Sun, NeXT, DEC, Silicon Graphics, HP, and IBM RS6000 workstations,
Stardent Titan, BBN GP-1000 and TC-2000, Intel IPSC/860, Intel Touchstone
Delta, Alliant Campus, and Thinking Machines' CM-5.  It will soon be ported to
to the Intel Paragon.  It is not difficult to port to new systems.  Although
p4 tries to be completely portable, there are a small number of
specific exceptions (\xref{Machine-Specific Notes}) that may need to be taken
into account on a given machine.

You can obtain the complete distribution of p4 by anonymous ftp from
\code{info.mcs.anl.gov}.  Take the file \file{p4.tar.Z} from the
directory \file{pub/p4}.  The distribution contains all source code,
installation instructions, this reference manual, and a collection of
examples in both C and Fortran. \code{Alog} is included in the
distribution with p4.  The file \file{upshot.tar.Z} contains display
facilities that can be used with p4 and other systems.

To ask questions about p4, report bugs, contribute examples, etc., you can
send mail to \code{p4@mcs.anl.gov}.

The current release is version 1.2.  You can check which version of the source
code you have by looking at the file \file{lib/p4_patchlevel.h} in the
distribution.  You can check which version of the object code you have linked
to by running any p4 program with the command-line option \code{-p4version}
(\xref{Command-Line Arguments}).

Salient features of the current release of p4 include: 
\begin{itemize}
\item \code{xdr} support for heterogeneous networks
\item manual enhanced and converted to latex format
\item Emacs info version of the manual for on-line help
\item SYSV IPC support added for several machines (for shared-memory
  multiprocessing on workstations that support multiple processors)
\item instrumentation added for automatic logging/tracing
\item automatic or user control of message-passing/buffer-management
\item high-resolution clock support for several machines
\item error/interrupt handling
\item an optional secure server for quick startup
\end{itemize}

A useful companion system is the \code{alog/upshot} logging and X-based trace
examination facility.  (\xref{Creating Logfiles for Upshot}.)


\node Structure of the Distribution Directory,Installing p4,Introduction,Top

\section{Structure of the Distribution Directory}
\cindex{directory structure}

The p4 source code distribution contains the following files and
subdirectories:

\cindex{OPTIONS}
\begin{description}
\item[Makefile] The makefile for making the p4 system, doing the installation,
  and making makefiles for user applications.
\item[OPTIONS] A file controlling various compile-time options, such as
  whether System V shared-memory operations are to be enabled, whether system
  debug message printing is to be enabled, and whether automatic
  instrumentation of p4 operations for the \code{upshot} logging and tracing
  program is to be done.
\item[README] General instructions.
\item[alog] Source code for the ALOG tracing package.
\item[bin] Scripts for starting and killing servers, killing runaway p4
  processes, merging \code{upshot} logfiles, and other useful utilities.
\item[contrib] Examples contributed by p4 users.
\item[contrib_f] Fortran examples contributed by users.
\item[doc] The man page, together with this manual and supporting files.
\item[include] The include directory for making p4 applications.  Most of
  these are (hard) links into the \code{lib} directory.
\item[lib] The source code for the p4 system.
\item[lib_f] The Fortran interface for p4.  
\item[messages] A basic set of message-passing examples in C.
\item[messages_f] A basic set of message-passing examples in Fortran.
\item[monitors] A basic set of shared-memory examples in C.
\item[servers] The secure and insecure servers.
\item[usc] The portable microsecond clock routines.
\item[util] Assorted supporting files, particularly for making the p4
  distribution. 
\end{description}

\node Installing p4,Getting Started,Structure of the Distribution Directory,Top

\section{Installing p4}
\cindex{installation}

In this section we describe how to install the p4 library, either for
your own personal use or for the use of everyone at your site.  In the
first case you do not need any super-user privileges.  In the second
case, you may or may not, depending on how things are configured at
your site.  We also describe how to install and run the examples that
come with p4, the online help system (this manual as an emacs info-file)
and how to build a working directory for your own programs yet share
the installed copy of p4 with other users.


\begin{menu}
* Installing the p4 System::
* Installing the Documentation::
* Examples included with the Distribution::
\end{menu}

\node Installing the p4 System,Installing the Documentation,	,Installing p4

\subsection{Installing the p4 System}

To build p4, position yourself in the \code{p4} directory and type:
\begin{example}
make all MACHINE=<machine>
\end{example}
\noindent
where <machine> is one of the machine names listed in
\file{p4/util/machines}, currently:
\cindex{machines}

\begin{ifinfo}
\begin{description}
\item[SUN] Sun-3, Sun386i, Sparc-1, or Sparc-2 workstations

\item[HP] HP workstations

\item[DEC5000] Dec 5000 workstations

\item[NEXT] 68030- or 68040-based NeXT workstations

\item[RS6000] IBM RS 6000 series workstations

\item[IBM3090] IBM 3090 running IBM's version of UNIX, AIX

\item[BALANCE] Sequent Symmetry shared-memory multiprocessor

\item[SYMMETRY] Sequent Symmetry shared-memory multiprocessor

\item[SYMMETRY_PTX] Sequent Symmetry shared-memory multiprocessor PTX OS

\item[MULTIMAX] Encore Multimax shared-memory multiprocessor

\item[GP_1000] BBN GP-1000

\item[TC_2000] BBN TC-2000

\item[TC_2000_TCMP] BBN TC-2000 with the TCMP message-passing library

\item[IPSC860] Intel IPSC/860 (nodes only)

\item[IPSC860_SOCKETS] Intel IPSC/860 with socket libraries on the nodes

\item[DELTA] Intel DELTA 

\item[TITAN] Stardent Titan

\item[SGI] Silicon Graphics workstations

\item[CRAY] Cray X/MP

\item[FX8] Alliant FX/8

\item[FX2800] Alliant FX/2800 or FX/800

\item[FX2800_SWITCH] Alliant FX/2800 or FX/800, with CAMPUS HiPPI switch

\item[KSR] Kendall Square KSR-1

\item[CM-5] Thinking Machines' CM-5
\end{description}

\end{ifinfo}

\begin{iftex}
\begin{tex}
\begin{tabular}{l l}
{\tt SUN} & Sun-3, Sun386i, Sparc-1, or Sparc-2 workstations \\
{\tt HP} & HP workstations \\
{\tt DEC5000} &  Dec 5000 workstations \\
{\tt NEXT} &  68030- or 68040-based NeXT workstations \\
{\tt RS6000} &  IBM RS 6000 series workstations \\
{\tt IBM3090} &  IBM 3090 running IBM's version of UNIX, AIX \\
{\tt BALANCE} &  Sequent Symmetry shared-memory multiprocessor \\
{\tt SYMMETRY} &  Sequent Symmetry shared-memory multiprocessor \\
{\tt SYMMETRY\_PTX} &  Sequent Symmetry shared-memory multiprocessor PTX OS \\
{\tt MULTIMAX} &  Encore Multimax shared-memory multiprocessor \\
{\tt GP\_1000} &  BBN GP-1000 \\
{\tt TC\_2000} &  BBN TC-2000 \\
{\tt TC\_2000\_TCMP} &  BBN TC-2000 with the TCMP message-passing library \\
{\tt IPSC860} &  Intel IPSC/860 (nodes only) \\
{\tt DELTA} &  Intel DELTA  \\
{\tt TITAN} &  Stardent Titan \\
{\tt SGI} &  Silicon Graphics workstations \\
{\tt CRAY} &  Cray X/MP \\
{\tt FX8} &  Alliant FX/8 \\
{\tt FX2800} &  Alliant FX/2800 or FX/800 \\
{\tt FX2800\_SWITCH} &  Alliant FX/2800 or FX/800, with CAMPUS HiPPI switch \\
{\tt KSR} &  Kendall Square KSR-1 \\
{\tt CM5} &  Thinking Machines' CM-5 \\

\end{tabular}
\end{tex}
\end{iftex}

\noindent
For example:
\begin{example}
make all MACHINE=SYMMETRY
\end{example}
\noindent
The \code{all} is optional, for example
\begin{example}
make MACHINE=SEQUENT
\end{example}
\noindent
This will create a machine-dependent \file{Makefile} in each subdirectory,
make the p4 library, and compile and link a subset of the examples.

To add a new machine type, or to change the characteristic parameters
associated with an existing one, you can edit the file 
\file{p4/util/defs.all}.

To save disk space, various intermediate object files can be removed with
\begin{example}
make clean
\end{example}
\noindent
The system can be restored to its original, machine-independent state with
\begin{example}
make realclean
\end{example}
\noindent
Note that this removes the machine-dependent Makefiles in each directory, so
the operation is not idempotent.

\noindent
It is also possible to install (or clean) only some of the directories:
\begin{example}
make all MACHINE=SUN DIRS=messages
make clean DIRS='monitors messages'
\end{example}
\noindent
To install only the Makefiles in all subdirectories, use:
\begin{example}
make makefiles MACHINE=<machine>
\end{example}
\noindent To install the necessary library and include files in a
directory everything that is needed to compile and link p4 programs, 
do:

\begin{example}
make install INSTALLDIR=<dir>
\end{example}
\noindent This will create a p4 directory in \code{<dir>}, build a
minimal set of directories, copy the relevant \file{.a} and \file{.h} files
into it, and test the installation by mking a small set of examples.

\cindex{testing}
\xref{Getting Started} for instructions on how to run some example
programs after you have installed p4.
\node Installing the Documentation,Examples included with the Distribution,Installing the p4 System,Installing p4

\subsection{Installing the Documentation}
\cindex{documentation}
\cindex{man pages}
\cindex{online help}
\cindex{emacs Info}

The directory \file{p4/doc} contains this manual as well as files that
require installation.  This manual was prepared with the
\code{latexinfo} package from GNU emacs; thus it can be made available online
through \code{info}.  The files in \file{p4/doc} are:

\begin{description}
\item[p4.tex] the latex source for this manual, which uses the latexinfo style

\item[latexinfo.sty] the sytle file needed to latex this manual

\item[p4.info] the \code{info} version of this manual, ready to be put
into the directory where \code{info} files are kept at your site.  Check
the value of your \code{emacs} variable \var{Info-directory}.

\item[p4.txt] plain ascii text of the manual, in case nothing else works.

\item[p4.1] unix man page for the p4 library

\item[p4f.1] unix man page for the Fortran interface to p4

\end{description}

The Postscript version of this manual is available by anonymous \code{ftp}
from \code{info.mcs.anl.gov}, in the directory \file{pub/p4}.  The file to
get (in binary mode) is \file{p4man.ps.Z}.

\node Examples included with the Distribution,	,Installing the Documentation,Installing p4

\subsection{Examples included with the Distribution}
\cindex{examples}

A good way to see how various p4 functions are used is to look at the example
programs included in the distribution.  The \file{p4/monitors} directory
contains shared-memory examples written in C that use monitors, including one
instrumented with ALOG.  The \file{p4/messages} subdirectory contains
message-passing examples written in C.  The programs in \file{p4/messages_f}
are Fortran message-passing examples, and the \file{p4/contrib} and
\file{p4/contrib_f} directories contain a number of miscellaneous examples
contributed by users.  In each directory there is a \file{README} that
describes the individual examples.

\node Getting Started,Specifying Processes in the Procgroup File,Installing p4,Top

\section{Getting Started}
\cindex{getting started}

The easiest way to get started with p4 is to play with some of the
sample programs provided with the system.

\begin{menu}
* A Message-Passing Example::
* Analysis of the Program::
\end{menu}

\node A Message-Passing Example,Analysis of the Program,	,Getting Started

\subsection{A Message-Passing Example}

We will begin with a message-passing example in the sub-directory named
\file{p4/messages}.  The code for the program is in the files \file{sr_test.c}
and \file{sr_user.h}.


 \subsection{Program Description}

As the name implies, this program is an example of p4's send/receive
functionality.  Briefly, it is a simple program that runs a master
process and some slave processes.  The master and the set of slaves
form a ring of processes in which the master reads a message from stdin
and sends a copy of the message to the first slave, which passes it on;
the last slave passes the message back to the master.  If the master
receives an undamaged copy of the message, it assumes that all went
well, and reads another message.  Note that the ring of processes is a
logical structure in which each process assumes that its
predecessor in the ring is the process with the next lower id, and
its successor is the process with the next higher id.  The master
has id 0 (zero) and has the process with the largest id as its
predecessor.


\node Analysis of the Program,	,A Message-Passing Example,Getting Started

\subsection{Analysis of the Program}

The first executable p4 statement in a program should be:
\begin{example}
p4_initenv(&argc,argv); 
\end{example}
\noindent
This initializes the p4 system and allows p4 to extract any command
line arguments passed to it, e.g. debugging parameters.  

Similarly, the last executable p4 statement in a program should be:
\begin{example}
p4_wait_for_end(); 
\end{example}
\noindent
This waits for termination of p4 processes and performs some cleanup
operations.

The procedure \code{p4_get_my_id} returns the unique integer id assigned
to the calling process by p4.

The statement:
\begin{example}
p4_create_procgroup();
\end{example}
\noindent
reads a procgroup file that the user builds and creates the set of
slaves described in that file.  Obviously this statement must be
executed before any slaves can be assumed to exist.  This procedure
is the method you must use to create processes that do message-passing.

The procedure \code{p4_clock} returns an integer that represents
wall-clock time in milliseconds.  It is typically used to retrieve the
time before and after some work, the difference representing the time to
do that work.  Note that there is also a \code{p4_ustimer} that is useful on
those machines that support a microsecond timer.

The procedures \code{p4_send} and \code{p4_sendr} are two of several 
p4 procedures that are available for sending messages to other processes.
They take as arguments the message type, the id of the "to" process,
the address of the message, and the message length.

The procedure \code{p4_recv} receives a message from another process and
sets the values of all four parameters.  \code{P4_recv} will automatically
retrieve a buffer in which to place a received message, thus
\code{p4_msg_free} may be called to free that buffer when it is no longer
needed.

The procedure \code{p4_num_total_slaves} is one of several procedures that
the user can invoke to determine information about the current execution.

To run this program, you need to create a procgroup file that describes
where all slave processes are to be executed 
(\xref{Specifying Processes in the Procgroup File}).  We will assume that
you have an example procgroup file (named \file{procgroup}) in the 
\file{p4/messages} directory, and can run \code{sr_test} by merely typing:
\begin{example}
sr_test
\end{example}
\noindent
If the procgroup file is elsewhere, then you must type:
\begin{example}
sr_test -pg  {\it pathname_of_procgroup_file}
\end{example}

\noindent
Another example that is made by default is the program \code{systest}.  It
tests a number of the message-passing features of p4.

\node Specifying Processes in the Procgroup File,Developing a Simple p4 Program,Getting Started,Top

\section{Specifying Processes in the Procgroup File}
\cindex{procgroup files}

The procgroup file is the only portion of the interface that is very
likely to change through multiple versions of p4.  As new architectures
are supported, it is hoped that we can merely alter the procgroup file
format to reflect any new features.  (Of course new procedure calls may 
also be required, but existing procedure calls will remain unchanged 
when possible).

The current format of a procgroup file is as follows:
\begin{example}
local n [full_path_name] [loginname]
remote_machine n full_path_name [loginname]
.
.
.
\end{example}
\noindent

On cube and mesh architectures, the program is started via some
special command executed from the host machine.  In such cases, the
procgroup file name can be specified to the special command line along
with the program name (see for example the \code{runcube} and \code{rundelta}
shell scripts in the \file{p4/messages} subdirectory).  In those cases 
where no special command is required, no special handling is required 
for the procgroup filename.

The first line of a procgroup file must be ``local n'' where n is the
number of slave processes that are in the same cluster as the master.
The full path name on the ``local'' line is ignored on machines other
than cube and mesh machines.  The subsequent lines contain either
three or four fields:
\begin{enumerate}
\item
the name of a remote machine on which slave processes are to be created.
\item
the number of slaves that are to be created on that machine, 
i.e. be in the same cluster (note that on machines that support it, 
the processes in a cluster will share memory)
\item
the full path name of the executable slave program
\item
optionally, the user login name on the remote machine, if different from 
that on the host machine.
\end{enumerate}

As an example, let's assume that you have a network of three Sun 
workstations named sun1, sun2, and sun3.  We will also assume that you 
are working on sun1 and plan to run a master process there.  
If you would like to run one process on each of the other Suns, then you 
might code a procgroup file that looks like:

\begin{example}
    # start one slave on each of sun2 and sun3
    local 0 
    sun2  1  /home/mylogin/p4pgms/sr_test
    sun3  1  /home/mylogin/p4pgms/sr_test
\end{example}

Lines beginning with \code{#} are comments.

Next, let's assume that you have a Sequent Symmetry (named symm) and an
Encore Multimax (named mmax).  We will also assume that you are working
on symm, and plan to run the master there.  If you would like to run
two processes on symm (in addition to the master) and two on mmax, then 
you might code a procgroup file that looks like:

\begin{example}
    local 2 
    mmax  2  /mmaxfs/mylogin/p4pgms/sr_test
\end{example}

P4 also permits you to treat the symmetry as a remote machine even when 
you are running the master there.  Thus, you might code a procgroup file 
as follows:

\begin{example}
    local 2 
    symm  2  /symmfs/mylogin/p4pgms/sr_test
    mmax  2  /mmaxfs/mylogin/p4pgms/sr_test
\end{example}

In this example, there are seven processes running.  Five of the
processes are on symm, including the master.  Two of the processes on
symm are in the master's procgroup and two are running in a separate
procgroup as if they were on a separate machine.  Of course, the last
two are running on mmax.

Some notes about the contents of the procgroup file should be made at
this point.  First, the value of \code{n} on the local line can be zero,
i.e. the master may have no local slaves.  Second, the local machine
may be treated as if it is a remote machine by merely entering it in
some line as a remote machine.  Third, a single machine may be treated
as multiple remote machines by having the same remote machine name
entered on multiple lines in the procgroup file.  Fourth, if a single
machine is listed multiple times, those processes specified on each
line form a single cluster (share memory).  Fifth, the cluster size
specified for a uniprocessor should be 1, because all slaves in a
cluster are assumed to run in parallel and to share memory.

We refer to the original (master) process as the ``big master''.  The
first process created in each cluster is the ``remote master'' or the
``cluster master'' for that cluster.  
All p4-managed processes (see the procedure \code{p4_create_procgroup}) 
have unique integer id's beginning with 0.
The processes within a cluster are numbered consecutively.


\node Developing a Simple p4 Program,Command-Line Arguments,Specifying Processes in the Procgroup File,Top

\section{Developing a Simple p4 Program}

The real fun associated with any computing environment arrives when you
actually type in a program and run it yourself.  We will assume that
you have successfully installed p4 on your own system and are ready to
write a small program, compile it, and run it.

\begin{menu}
* A Minimal Example::   
* A More Complicated Example::
\end{menu}

\node A Minimal Example,A More Complicated Example,	,Developing a Simple p4 Program

\subsection{A Minimal Example}

We will start with a tiny program in which the slave processes do no work, and
then expand its capabilities.  Edit a file called \file{p4simple.c} and type:

\begin{example}

    #include "p4.h"

    main(argc,argv)
    int argc;
    char **argv;
    \{
        p4_initenv(&argc,argv);
        if (p4_get_my_id() == 0)
            p4_create_procgroup();
        slave();
        p4_wait_for_end();
    \}

    slave()
    \{
        printf("Hello from %d\verb+\n+",p4_get_my_id());
    \}

\end{example}

This is one of the simplest p4 programs that you can write.  Let's
examine it.  The \code{#include "p4.h"} statement must appear in all
programs that use any p4 features.  The procedure \code{p4_initenv}
must be invoked before any other p4 procedures, and
\code{p4_wait_for_end} must be invoked after all p4 processing is
completed.  The \code{p4_get_my_id} returns a unique integer id for
each process, beginning with 0.  The procedure
\code{p4_create_procgroup} should only be invoked once (by process 0),
and is responsible for creating all other processes.  The way in which
\code{p4_create_procgroup} determines how many other processes there
should be, and where they should run, will be discussed shortly.

All processes that this program executes invoke the slave procedure,
including process 0.  Thus, in this program, the master process acts
just like all other processes once it gets the environment
established.

To understand how things get started, let's consider two separate
situations.  In the first situation, all processes are running on a
single machine.  Then, when process 0 starts, it executes the
\code{p4_create_procgroup} procedure to start all other slaves.  The other
slaves are started on the same machine by means of a UNIX {\em fork},
and they immediately invoke a procedure named slave.  Thus, these local
slaves do not ever execute the main procdure.

In the second situation, there may be slaves running both on the same
machine as process 0, and slaves running on other machines as well.
In this situation, the first slave running on a remote machine will
need to execute the main procedure.  It will discover that it is not
process 0.  However, as part of initialization, process 0 will
direct it to fork any additional slaves required on the same machine.

\cindex{slave subroutine}
In some ways, the above example can be used a a prototype for all p4 programs,
just by varying the content of the \code{slave} routine.  The routine name
\code{slave} is built into p4 in order to enable the same code to be run on
shared-memory machines via \code{fork} or on remote machines via \code{rsh}.
Different wrappers are automatically put around the \code{slave} code.  These
wrappers depend on the user code being called \code{slave}.  Thus your p4
program must contain a subroutine called \code{slave} in order to link
properly. 

\node A More Complicated Example,	,A Minimal Example,Developing a Simple p4 Program

\subsection{A More Complicated Example}

Now, let's make the slave process a little bit more interesting.  Let's
assume that we have \code{nprocs} slaves with ids 
0, 1, 2, ... \code{nprocs} -1.  And, we want to write a program in which
every process sends a single message to every other slave, and then
receives a message from every other slave.  We might alter the code for
the slave procedure to be the following:

\begin{example}
    slave()
    \{
        char *incoming, *msg = "hello";
        int myid, size, nprocs, from, i, type;

        myid = p4_get_my_id();
        nprocs = p4_num_total_ids();
        for (i=0; i < nprocs; i++)
        \{
            if (i != myid)
                p4_send(100, i, msg, strlen(msg)+1);
        \}
        for (i=0; i < nprocs - 1; i++)
        \{
            type = -1;
            from = -1;
            incoming = NULL;
            p4_recv(&type,&from,&incoming,&size);
            printf("%d received msg=:%s: from %d\verb+\n+",myid,incoming,from);
            p4_msg_free(incoming);
        \}
    \}
\end{example}

This program demonstrates several features of p4's support for
message-passing.  Before we get into the specifics however, let's
examine the overall logic of the program.  Each process determines its
own id and the total number of processes executing in this run
(including process 0).  Then, in the first for-loop, each process sends
a single message to each of the other processes.  Finally, in the
second for-loop, each process receives a message from each of the other
processes.

The \code{p4_send} call requires 4 arguments:
\begin{itemize}
\item a message type (arbitrarily chosen to be 100 here)
\item the id of the process to receive the message
\item the message itself
\item the size of the message
\end{itemize}

The use of \code{p4_recv} is slightly more complicated.  First, we
assign -1 to each of the parameters type and from.  This is done
because -1 represents a wildcard value indicating we are willing to
receive a message of any type from any process.  Here, we could have
coded type to be 100, and specified from equal to the value of i each
time through the loop (skipping our own id).  By setting incoming to
NULL, we have also indicated to \code{p4_recv} that we do not have a
buffer in which to place the received message, so \code{p4_recv} should
obtain a buffer for us and place the message in that buffer.
\code{p4_recv} treats these three parameters as both input and output
values.  Thus, it alters the value of each such that type and from
indicate the type of message received and the id of the process that
sent it.  The value of \code{incoming} is altered to point to the
buffer where the message was placed.  The \code{size parameter} is
strictly an output parameter and iindicates the size of the received
message.  It is possible for the user to provide his own buffer; this
will be demonstrated later.

\cindex{deallocating buffers}
Finally, note that \code{p4_msg_free} frees the message buffer obtained by
\code{p4_recv.} The procedure \code{p4_msg_free} should be called only after
the contents of the message are no longer needed.  \code{P4_msg_free} should
be used to free these buffers because, although a user only sees the data
portion of a message, p4 internally represents a message as a structured data
item.

To compile and link this program for execution, you need to create a
makefile.  We will assume that you have installed p4 in
\file{/usr/local/p4} and that you have typed the program above into a
file name \file{p4simple.c} in the directory
\file{/home/mylogin/p4pgms}.

To build your makefile, copy the file
\begin{example}
    /usr/local/p4/messages/makefile.proto
\end{example}
\noindent
into your working directory.  This is a prototype makefile that
contains machine-independent information, and which p4 can use to build a
machine-specific makefile for your program.  This prototype makefile contains
information about several sample programs that demonstrate
message-passing in p4.  If you edit this file, you will see information
for making a program named \code{sr_test}.  Do a global change of
\code{sr_test} to \code{p4simple}.  You should also change the value of
\code{P4_HOME_DIR}.  It should contain the full
pathname of the p4 system, e.g. \file{/usr/local/p4}.  Now change
directories to \file{/usr/local/p4} and type:

\begin{example}
make makefiles MACHINE=<machine_type> DIRS=/home/mylogin/p4pgms
\end{example}
\noindent
where \code{<your_machine_type>} is the machine type that you specified when
you installed p4 on your machine.  Now, you should be able to change
back to your directory and see a file named Makefile there.  
You should then be able to type:

\begin{example}
    make p4simple
\end{example}

There is one last piece missing before you can execute your program.
Recall that \code{p4_create_procgroup} needs to know how many processes
to start and where to start them; it reads a file (called a
\dfn{procgroup file}) to gather this information.  p4 always assumes
that you have a master process, and that you describe the slave
processes (process groups) in the procgroup file.  You can name a
procgroup file any name you choose, but \code{procgroup} is the default
name.  The information contained in procgroup files can get fairly
involved, but if you have a computer that supports shared memory among
processes, then you can code a very simple example at first.

Let us suppose first that you want to run your program on a network of
workstations.  Then your procgroup should look something like:

\begin{example}
    local O
    some.network.machine 1 /home/me/p4progs/p4simple
\end{example}

This file indicates that you wish to run only the master on the local machine
(the one you are logged into when you execute the program) and one slave on
the machine \code{some.network.machine}.

Now, all you have to do to run your program is type:

\begin{example}
    p4simple
\end{example}

You should see a line printed each time a process receives a message
from another process (on some machines, there may be a restriction that
only one process can do I/O, however such restrictions are not
common).  Experiment by changing the number of slaves indicated in the
procgroup file.

You may notice that even a small p4 program becomes large when linked
with the p4 library.  You might consider using \code{strip} to reduce
the size or removing \code{-g} from the CFLAGS in the makefile.

\node Command-Line Arguments,The p4 Function Library,Developing a Simple p4 Program,Top

\section{Command-Line Arguments}
\cindex{command-line arguments}

The command-line arguments to a p4 program are all optional.
\begin{example}
-help           get this information and the version number, then exit
-pg             procgroup_file
-dbg            set debug level 
-rdbg           set remote debug level
-gm             set globmemsize
-dmn            provide local domainname
-outfile        output file for master
-routfile       output file prefix for remote masters
-ssport         port# private port number for secure server
-p4log          enable internal p4 logging by alog
-p4version      print the current p4 version number
\end{example}


\node The p4 Function Library,p4 Functions for Managing Processes and Clusters,Command-Line Arguments,Top

\section{The p4 Function Library}

\begin{menu}
* Overview of the Library::
* Return Codes from p4 Functions::
\end{menu}

\node Overview of the Library,Return Codes from p4 Functions,	,The p4 Function Library

\subsection{Overview of the Library}

In the following sections, we provide details for each p4 function in the
library.  The procedures are gathered into the following groups:

\begin{itemize}
\item Functions for managing processes and clusters
\item Functions for message passing
\item Functions for shared memory
\item Functions for timing p4 programs
\item Functions for debugging p4 programs
\item Miscellaneous functions
\item Fortran interface functions
\end{itemize}

\node Return Codes from p4 Functions,	,Overview of the Library,The p4 Function Library

\subsection{Return Codes from p4 Functions}
\cindex{return codes}

Most p4 functions return -1 if an error occurs.  Some, however, call the
function \code{p4_error} when severe errors occur.  This function prints a
message and then attempts to terminate all of the user's processes
\xref{Functions for Debugging p4 Programs}.

\node p4 Functions for Managing Processes and Clusters,Functions for Message Passing,The p4 Function Library,Top

\section{p4 Functions for Managing Processes and Clusters}

In some situations a p4 procedure will give an error message and then
exit.  This is typically done as a result of a failed system call and
handled by calling the p4 procedure named \code{p4_error} that examines the
return values from socket procedures, etc.  Most of the time however,
the procedures simply return a value.  Some of the procedures return no
value and thus are declared to return \code{VOID}.  Some of the
procedures return either a pointer to a character string or \code{NULL};
\code{NULL} indicates an error.  The remaining procedures return an
integer value; (-1) indicates an error.


\begin{menu}
* Functions for Process Management::
* Functions for Cluster Management::
\end{menu}

\node Functions for Process Management,Functions for Cluster Management,	,p4 Functions for Managing Processes and Clusters

\subsection{Functions for Process Management}
\cindex{process management}

In this section we describe the p4 functions needed for basic creation and
termination of processes.

\findex{p4_initenv}
\begin{example}
int p4_initenv(argc,argv)
int *argc;
char **argv;
\end{example}
\noindent 
should be called by your program before an attempt is made to use
any p4 procedures or data areas.  We suggest making it the first executable
statement in your program.  \code{p4_initenv} parses the command line
arguments and extracts the ones intended for p4 ignoring all others (see the
discussion of command line arguments).  Note that you pass the address of
\code{argc} to \code{p4_initenv} so that it can actually remove its own
arguments before your program looks at them.

\findex{p4_create}
\begin{example}
int p4_create(fxn)
int (*fxn)();

\findex{p4_create_procgroup}
int p4_create_procgroup()
\end{example}
\noindent
There are two procedures that you can use to create processes in p4,
\code{p4_create_procgroup} and \code{p4_create}.  Processes created via
\code{p4_create} are said to be ``user-managed'' whereas those created
by \code{p4_create_procgroup} are ``p4-managed''.  The p4-managed
processes are automatically assigned unique id's (beginning with 0 for
the big master), they have message queues allocated for them so that
they can do message-passing, and they are able to run either on a
shared-memory multiprocessor with the creating process or they can run
on a separate machine.  Processes created via \code{p4_create} do not
have any of these advantages.  They must develop their own id's, they
cannot do message-passing, and they can only run on a shared-memory
multiprocessor with the creating process.  The only disadvantage of
\code{p4_create_procgroup} is that you must build a \file{procgroup}
file describing the set of required slave processes before the master
program begins execution.  This eliminates the possibility of
determining late in the execution exactly how many processes you want
to use to solve a problem.  Generally, this is not a problem,
especially since we can combine \code{p4_create_procgroup} and
\code{p4_create} in the following way: You can use
\code{p4_create_procgroup} to develop a network of processes that talk
to each other via messages.  Each of those processes can create further
processes to help it out as necessary.  The original set of processes
communicate with their local slaves through shared data areas and with
each other via message-passing.

\code{p4_create} receives one argument that is a pointer to a function.  It
creates a single new process that executes the indicated function.  The
new process may share data areas (in shared memory) with the parent
process.  However, the new process is not managed by the p4 system in
the sense that it is not assigned an id, it cannot pass messages, etc.
The only p4 procedure that deals with user-managed slaves is \code{p4_create}.
No other procedures are even aware of their existence.

\code{p4_create_procgroup} reads your \file{procgroup} file to determine the 
number of slave processes to create and where they are to be placed.  It
builds a procgroup table that describes all created processes and gives
a copy of the table to each process.  The processes then use the table
to discover how to communicate with each other (processes in a cluster
can send messages directly through shared memory or some other
vendor-specific mechanism), others communicate via sockets). 
An alternative method is to build the table in memory yourself and use
\code{p4_startup}.

The effect of \code{p4_create_procgroup} can be obtained in another way if a
system would prefer to use its own way of specifying the locations of
processes.  A user may allocate the procgroup data structure and then fill it
in ``by hand'' rather than by reading a file in p4 procgroup format.  The
following procedures support this method of starting processes.

\findex{p4_alloc_procgroup}
\begin{example}
struct p4_procgroup *p4_alloc_procgroup()
\end{example}
\noindent
allocates a procgroup data structure of the form described in \code{p4.h}.
The formats of individual entries (\code{p4_procgroup_entry}) are given there
as well. 

\findex{p4_startup}
\begin{example}
int 4_startup(pg)
struct p4_procgroup *pg;
\end{example}
\noindent
starts processes as specified by an an already-created procgroup data
structure allocated by \code{p4_alloc_procgroup} and filled in by the user
using the structures \code{p4_procgroup_entry} and \code{p4_procgroup}.


\findex{p4_wait_for_end}
\begin{example}
VOID p4_wait_for_end()
\end{example}
\noindent
is the p4 termination/cleanup procedure that you should invoke at the
end of every execution of a program that uses p4.  
It does some termination processing and then waits for slave processes 
to end.

\findex{p4_get_my_id}
\begin{example}
int p4_get_my_id()
\end{example}
\noindent
returns an integer value representing the id of the process assigned by
the p4 system.  If the process is not a p4-managed process, the value
(-1) is returned.

\findex{p4_num_total_ids}
\begin{example}
int p4_num_total_ids()
\end{example}
\noindent
returns an integer value indicating the total number of ids started
by p4 in all clusters, including the big master and all remote masters.

\findex{p4_num_total_slaves}
\begin{example}
int p4_num_total_slaves()
\end{example}
\noindent
returns an integer value indicating the total number of processes started
by p4 in all clusters, including all remote masters but not the big master.

\node Functions for Cluster Management,	,Functions for Process Management,p4 Functions for Managing Processes and Clusters

\subsection{Functions for Cluster Management}
\cindex{cluster management}

The p4 system supports the \dfn{cluster} model of parallel computation, in
which subsets of processes share memory with one another, with the clusters
communicating via messages.  A procgroup file for a program written for the
cluster model might look like this:

\begin{example}
    local 4
    alliant1.abc.edu     5 /home/me/myprog
    alliant2.abc.edu     5 /home/me/myprog
    encore.somewhere.edu 5 /usrs/me/myprog
\end{example}

This would specify a total of 20 processes, 5 (including the master) running
on the local machine (here assumed to be capable of supporting five processes
that share memory) together with 5 slaves each on three other shared-memory
machines.

\findex{p4_get_cluster_ids}
\begin{example}
VOID p4_get_cluster_ids(start,end)     
int *start;
int *end;
\end{example}
\noindent
receives pointers to two integers.  It places the p4-assigned id's of
the first and last ids within the current cluster into the two
arguments (including the remote master).

\findex{p4_get_my_cluster_id}
\begin{example}
int p4_get_my_cluster_id()
\end{example}
\noindent
returns a unique id (relative to 0) within a cluster of p4-managed
processes.  Thus, a cluster master will always have a cluster id of 0.
It is not clear that a separate cluster id is really useful, but the
functionality is provided just in case.

\findex{p4_am_i_cluster_master}
\begin{example}
BOOL p4_am_i_cluster_master()
\end{example}
\noindent
returns a BOOL value indicating whether the invoking process is the
``cluster master'' process within its cluster.

\findex{p4_num_cluster_ids}
\begin{example}
int p4_num_cluster_ids()
\end{example}
\noindent
returns an integer value indicating the number of ids in the current
cluster as started by \code{p4_create_procgroup}.  


\node Functions for Message Passing,Functions for Shared Memory,p4 Functions for Managing Processes and Clusters,Top

\section{Functions for Message Passing}
\cindex{message-passing functions}

P4 supports a set of send/receive procedures.  These procedures are
``generic'' in the sense that they do not know whether a message must
travel across a network or through shared memory, or via some other
mechanism.  They depend on a lower-level set of procedures that handle
local or network (remote) communications.  By default, the messages
are assumed to be typed.  If the user wishes to use untyped messages,
he can hide the typing by coding some very simple C macros that always
use a single message type.


\begin{menu}
* Explicit Sending and Receiving of Messages::
* Global Operations::   
\end{menu}

\node Explicit Sending and Receiving of Messages,Global Operations,	,Functions for Message Passing

\subsection{Explicit Sending and Receiving of Messages}

\cindex{sending messages}
\findex{p4_send}
\findex{p4_sendr}
\findex{p4_sendx}
\findex{p4_sendrx}
\findex{p4_sendb}
\findex{p4_sendbr}
\findex{p4_sendbx}
\findex{p4_sendbrx}
\begin{example}
p4_send(type,to,msg,len)
p4_sendr(type,to,msg,len)
p4_sendx(type,to,msg,len,datatype)
p4_sendrx(type,to,msg,len,datatype)
p4_sendb(type,to,msg,len)
p4_sendbr(type,to,msg,len)
p4_sendbx(type,to,msg,len,datatype)
p4_sendbrx(type,to,msg,len,datatype)

int type, to, len, datatype;
char *msg;
\end{example}
\noindent

Each of these procedures sends a message.  The \code{type} argument is
an integer value chosen by the user to represent a message type.  The
\code{to} argument is an integer value that specifies the p4-id of the
process that should receive the message.  The \code{len} argument
contains the length of the message to be passed.  Note that some of the
procedures have a ``b'' in their name, e.g.  \code{p4_sendb}.  These
procedures assume that the msg is in a buffer that the user obtained
earlier via a \code{p4_msg_alloc}; otherwise, the buffer is assumed to
be in the user's local space, and may cause the message to be copied
internally.  The procedures with an ``r'' in the name do not return
until an acknowledgement is received from the \code{to} process (the
``r'' stands for rendezvous).  Those procedures with an ``x'' in the
name take an extra argument (datatype) that specifies the type of data
in the message; these procedures will use that information to call XDR
for data conversion if the message is being passed to a machine of a
different architecture, i.e. where the internal representation may be
different.


\findex{p4_messages_available}
\begin{example}
BOOL p4_messages_available(req_type,req_from)
int *req_type,*req_from;
\end{example}
\noindent
returns a BOOL value indicating whether the process has any messages available
or not.  The parameters \code{req_type} and \code{req_from} are both pointers
to integers; they are used as {\em both} input and arguments.  On input,
\code{req_type} has a value that indicates the type of message that the user
wishes to check for availability (-1 indicates any type).  The variable
\code{req_from} is used similarly to indicate who a message is desired from.

\cindex{receiving messages}
\findex{p4_recv}
\begin{example}
int p4_recv(req_type,req_from,msg,len_rcvd)
int *req_type,*req_from,*len_rcvd;
char **msg;
\end{example}
\noindent takes four arguments.  The \code{msg} argument is a pointer to a
pointer to a \code{char}.  If this value is NULL, then p4 will allocate the
buffer for the message according to its length.  That is, one need not know
ahead of time the length of a message being received.  If this value is not
NULL, then it points to a p4 message buffer that the user has obtained via
\code{p4_msg_alloc}.  The \code{len_rcvd} argument is a pointer to an integer
that is assigned the length of the received message.  \code{Req_type} and
\code{req_from} are both pointers to integers; they are used as both input and
arguments.  On input, \code{req_type} has a value that indicates the type of
message that the user wishes to receive (-1 indicates any type).  It will
block until a message of that type is available.  \code{Req_from} is used
similarly to indicate who a message is desired from.  One important note about
this procedure is that it obtains the area in which to place a message, and
the user must explicitly free that area when finished with it (see
\code{p4_msg_free}).  There is an option available with \code{p4_recv} in
which the user can provide his own buffer rather than having p4 allocate it.
To do this, the user points \code{msg} to a buffer that he must obtain via a
call to \code{p4_msg_alloc} (see below).  Then he assigns \code{len_rcvd} a
value which is the length of that buffer.  In this case, \code{len_rcvd} is
both an input and output variable.  In addition, no \code{p4_msg_free} need be
performed if the same buffer is going to be re-used multiple times.

\cindex{allocating buffers}
\findex{p4_msg_alloc}
\begin{example}
char *p4_msg_alloc(len)
int len;
\end{example}
\noindent
obtains a pointer to a buffer area that can be used to receive a 
message.  This procedure should be used for this task because a 
message has hidden information which the user is unaware of and 
therefore should not use malloc to obatin the area.

\cindex{deallocating buffers}
\findex{p4_msg_free}
\begin{example}
VOID p4_msg_free(m)
char *m;
\end{example}
\noindent
frees the message pointed to by \code{m.}  This procedure should be used
for this task because a message has hidden information which the
user is unaware of and therefore cannot be freed by the user.


\node Global Operations,	,Explicit Sending and Receiving of Messages,Functions for Message Passing

\subsection{Global Operations}
\cindex{global operations}

P4 supports a number of operations for dealing with all processes at once.

\findex{p4_broadcastx}
\begin{example}
p4_broadcastx(type, data, data_len, data_type)
int type;
char *data;
int data_len, data_type;
\end{example}
\noindent

\findex{p4_broadcast}
\begin{example}
p4_broadcast(type, data, data_len)
int type;
char *data;
int data_len;
\end{example}
\noindent
provide the ability to broadcast messages like \code{p4_send} and
\code{p4_sendx}.

\findex{p4_global_op}
\begin{example}
p4_global_op(type,x,nelem,size,op,data_type) 
int type;
char *x;
int size, nelem;
int (*op)();
int data_type;
\end{example}
\noindent
where \code{op} is one of:
\begin{example}
p4_int_absmax_op()
p4_int_absmin_op()
p4_int_max_op()
p4_int_min_op()
p4_int_mult_op()
p4_int_sum_op()
p4_dbl_absmax_op()
p4_dbl_absmin_op()
p4_dbl_max_op()
p4_dbl_min_op()
p4_dbl_mult_op()
p4_dbl_sum_op()
p4_flt_absmax_op()
p4_flt_absmin_op()
p4_flt_max_op()
p4_flt_min_op()
p4_flt_mult_op()
p4_flt_sum_op()
\end{example}

\noindent
and \code{data_type} is one of \code{P4INT}, \code{P4LNG}, \code{P4FLT}, or
\code{P4DBL}.

This collection of routines provide the ability to do a variety of
global operations.  See the example programs in subdirectory
\file{p4/contrib}.  They apply the commutative operation \code{op} globally 
to \code{x} on an element-by-element basis and broadcast the result to 
all nodes.  That is each process ends up with

\begin{example}
   for (i=0; i<n; i++)
         x[i] = x[node 0][i] op x[node 1][i] op x[node 2][i] op ...
\end{example}
\noindent
\code{op} should be of the form

\begin{example}
     VOID op(char *x, char *y, int nelem)
     \{
         data_type *a = (data_type *) x;
         data_type *b = (data_type *) y;

         while (nelem--)
             *a++ operation= *b++;
     \}
\end{example}
\noindent
where \code{data_type} and \code{operation} are chosen appropriately.

The order in which nodes apply the operation is undefined (hence
\code{op} must be commutative). The communication may be internally
sub-blocked so the function \code{op} should not be hardwired to specific
vector lengths.

This is still a relatively primitive version, which gathers the necessary data
up a balanced binary tree and then uses \code{p4_broadcast} to send the
results back. 

\cindex{barrier}
\findex{p4_global_barrier}
\begin{example}
VOID p4_global_barrier(type)
int type;
\end{example}
\noindent
This procedure takes one argument which is the message type to be 
used for internal message-passing.  It causes the invoking process to
hang until all processes specified in the procgroup file have invoked 
the procedure.


\node Functions for Shared Memory,Functions for Timing p4 Programs,Functions for Message Passing,Top

\section{Functions for Shared Memory}
\cindex{shared memory functions}


\begin{menu}
* Managing Shared and Local Memory::
* Shared Memory Data Types::
* Monitor-Building Primitives::
* Some Useful Monitors::
\end{menu}

\node Managing Shared and Local Memory,Shared Memory Data Types,	,Functions for Shared Memory

\subsection{Managing Shared and Local Memory}

\findex{p4_malloc}
\begin{example}
char *p4_malloc(n)
int n;
\end{example}
\noindent
typically acts like the standard \code{malloc}, but may be rewritten for user 
systems that require different operation.

\findex{p4_free}
\begin{example}
VOID p4_free(p)
char *p;
\end{example}
\noindent
typically acts like the standard \code{free}, but may be rewritten for
user systems that require different operation.

\findex{p4_shmalloc}
\begin{example}
char *p4_shmalloc()
\end{example}
\noindent
acts like the standard \code{malloc} except will obtain shared memory on
machines that support sharing memory among processes.  Compare with
\code{p4_malloc}.

\findex{p4_shfree}
\begin{example}
VOID p4_shfree()
\end{example}
\noindent
frees memory obtained with \code{p4_shmalloc}.  Compare with \code{p4_free}.


\node Shared Memory Data Types,Monitor-Building Primitives,Managing Shared and Local Memory,Functions for Shared Memory

\subsection{Shared Memory Data Types}
\cindex{shared memory data types}
\cindex{monitor data types}
\cindex{data types for monitors}

The abstraction provided by p4 for managing data in shared memory is
\dfn{monitors}.  Good places to learn about the monitor concept in general are
\cite{pbh:architecture} and \cite{hoare:monitors}.  The specific approach
taken by p4 is described in \cite{lusk-overbeek:p4-book}.  P4 provides several
useful monitors (\code{p4_barrier_t}, \code{p4_getsub_monitor_t},
\code{p4_askfor_monitor_t}) as well as a general monitor type to help the user
in constructing his own monitors (\code{p4_monitor_t}).

\node Monitor-Building Primitives,Some Useful Monitors,Shared Memory Data Types,Functions for Shared Memory

\subsection{Monitor-Building Primitives}
\cindex{monitor primitives}

The following functions can be used to construct monitors.  A monitor so
constructed has the type \code{p4_monitor_t}.

\findex{p4_moninit}
\begin{example}
int p4_moninit(m,i)
p4_monitor_t *m;
int i;
\end{example}
\noindent
initializes the monitor pointed to by \code{m} and gives it \code{i} queues for
processes to wait on while they are blocked (see \code{delay}).  One queue is
sufficient for most purposes.  The queues are numbered beginning with 0.

\findex{p4_menter}
\begin{example}
VOID p4_menter(m)
p4_monitor_t *m;
\end{example}
\noindent
enter the monitor pointed to by m.  By the definition of a monitor,
access is restricted to a single process in the monitor at a time (if
everybody plays by the rules).

\findex{p4_mexit}
\begin{example}
VOID p4_mexit(m)
p4_monitor_t *m;
\end{example}
\noindent
exits the monitor pointed to by m.  You are of course assumed to have
previously entered that monitor.

\findex{p4_mcontinue}
\begin{example}
VOID p4_mcontinue(m,i)
p4_monitor_t *m;
int i;
\end{example}
\noindent
checks to see if there are any processes blocked on the \code{i}-th queue of
the monitor \code{m} and causes one of them to be released for entry to the
monitor if so.  If there are no such processes, the invoking process
simply exits.  Note that a process could have been blocked previously
by invoking the procedure \code{p4_mdelay}.  The queues are numbered 
beginning with 0.

\findex{p4_mdelay}
\begin{example}
VOID p4_mdelay(m,i)
p4_monitor_t *m;
int i;
\end{example}
\noindent
permits a process to delay itself on the \code{i}-th queue of monitor
\code{m} if the process wishes to release the monitor, but wants to be
waked up by another process later (via the procedure \code{p4_mcontinue}).
The queues are numbered beginning with 0.


\node Some Useful Monitors,	,Monitor-Building Primitives,Functions for Shared Memory

\subsection{Some Useful Monitors}
\cindex{monitors}

In this section we describe some of the specific monitors that are built into
the p4 library.  Each of them has its own pre-defined type, which can be
used to allocate storage for them, which should be in shared memory.
See the \file{p4/monitors} directory for examples.  A lock is itself a
monitor, with no extra delay queues.

\findex{p4_lock_init}
\begin{example}
VOID p4_lock_init(l)
p4_lock_t *l;
\end{example}
\noindent
initializes the lock \code{l}.  Must be used prior to any attempts to lock or
unlock \code{l}.

\cindex{lock}
\findex{p4_lock}
\begin{example}
VOID p4_lock(l)
p4_lock_t *l;
\end{example}
\noindent
blocks if the lock \code{l} is already locked, otherwise locks \code{l}
and proceeds.

\cindex{unlock}
\findex{p4_unlock}
\begin{example}
VOID p4_unlock(l)
p4_lock_t *l;
\end{example}
\noindent
unlocks the lock \code{l}.

\findex{p4_getsub}
\begin{example}
VOID p4_getsub(gs,s,max,nprocs)
p4_getsub_monitor_t *gs;
int *s,max,nprocs;
\end{example}
\noindent
is a procedure used to obtain the next value of a shared counter
(subscript).  It takes as its first argument, a pointer to a getsub
monitor that protects the shared counter.  It assigns the current value
of the counter to the integer that s points to, and then increments the
counter by 1.  \code{p4_getsub_init} initially sets the counter to 0.
When the counter passes the value \code{max}, all \code{nprocs}
processes are returned the value (-1) once, then the counter is reset to
0 for further use.

\findex{p4_getsubs}
\begin{example}
VOID p4_getsubs(gs,s,max,nprocs,stride)
p4_getsub_monitor_t *gs;
int *s,max,nprocs,stride;
\end{example}
\noindent
is like \code{p4_getsub} except that the counter is increased on each call by
\code{stride} instead of 1.


\findex{p4_getsub_init}
\begin{example}
int p4_getsub_init(gs)
p4_getsub_monitor_t *gs;
\end{example}
\noindent
initializes the getsub monitor pointed to by \code{gs}; this initialization
includes assigning a value of 0 to the counter that the monitor
protects.


The standard barrier synchronization pattern is expressed as a monitor.
There can be multiple barrier monitors, and one can wait for only some
of the processes at the barrier if this is desired.

\cindex{barrier}
\findex{p4_barrier}
\begin{example}
VOID p4_barrier(b,nprocs)
p4_barrier_monitor_t *b;
int nprocs;
\end{example}
\noindent
causes the executing process to hang until \code{nprocs} processes execute
a barrier instruction with a pointer to the same barrier monitor \code{b}
as an argument.

\findex{p4_barrier_init}
\begin{example}
int p4_barrier_init(b)
p4_barrier_monitor_t *b;
\end{example}
\noindent
initializes the barrier monitor \code{b}; this procedure should be invoked
before you attempt to use the monitor in any operations.

Finally, the \code{askfor} monitor functions like a general dispatcher of
work. 

\cindex{askfor monitor}
\findex{p4_askfor}
\begin{example}
int p4_askfor(af,nprocs,getprob_fxn,problem,reset_fxn)
p4_askfor_monitor_t *af;
int nprocs;
int (*getprob_fxn)();
VOID *problem;
int (*reset_fxn)();
\end{example}
\noindent
requests a new ``problem'' to work on from the problem pool.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) the number of processes that call this procedure
(with \code{af}) looking for work, (3) a pointer to the user-written procedure
that obtains a problem from the pool, (4) a pointer that is filled in
with the address of a user-defined representation of a problem to be
solved, and (5) a pointer to a user-written procedure to reset when all
problems in the pool are solved, in case the same monitor is re-used
for another set of problems later.  \code{p4_askfor} returns an integer
indicating whether a problem was successfully obtained or not:

\begin{example}
    -1     : program is terminating (some process called p4_progend)
     0     : a problem was obtained and ``problem'' points to it
     1     : problem solved by exhaustion, i.e. no more problems to get
     n > 1 : a process found a solution and called p4_probend with code n
\end{example}
\noindent
For a detailed discussion of the ``askfor'' monitor, see
\cite{lusk-overbeek:p4-book}.

\findex{p4_update}
\begin{example}
int p4_update(af,putprob_fxn,problem)
p4_askfor_monitor_t *af;
int (*putprob_fxn)();
VOID *problem;
\end{example}
\noindent
updates the problem pool being managed by the askfor monitor.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) a pointer to the user-written procedure that puts
problems into the pool, and (3) a pointer to a user-defined
representation of a problem to be put in the pool.  \code{Putprob_fxn}
should return 1 if it did indeed put a new problem into the pool, so
that any delayed processes should wake up and re-examine the pool (this
logic is handled by the \code{p4_askfor}) and 0 if upon entering the
monitor and examining its potential problem together with the data there
it decided not to add a new problem to the pool.  It can be assumed that
the ``putprob'' logic (defined by \code{putprob_fxn}) is executed inside
the monitor.

\findex{p4_askfor_init}
\begin{example}
int p4_askfor_init(af)
p4_askfor_monitor_t *af;
\end{example}
\noindent
initializes the askfor monitor \code{af}; this procedure should be invoked
before you attempt to use the monitor in any operations.

\findex{p4_probend}
\begin{example}
VOID p4_probend(af,code)
p4_askfor_monitor_t *af;
int code;
\end{example}
\noindent
allows the user process to mark a problem as solved early when
several processes are coordinating their activities via an askfor
monitor.  The code is an integer value that will be returned to all
processes when they ``askfor'' a new sub-problem to work on.

\findex{p4_progend}
\begin{example}
VOID p4_progend(af)
p4_askfor_monitor_t *af;
\end{example}
\noindent
allows a process to cause a return code of (-1) to be returned to all
processes using an askfor monitor.  This would typically be called by
a master process to indicate that no more problems are to be solved
and that all slave processes should terminate.


\node Functions for Timing p4 Programs,Functions for Debugging p4 Programs,Functions for Shared Memory,Top

\section{Functions for Timing p4 Programs}
\cindex{timing}
\cindex{clocks}
A small number of simple functions are available for accessing various
clocks and timers.  

\findex{p4_clock}
\begin{example}
int p4_clock()
\end{example}
\noindent
returns a value in milliseconds.  This is a wall-clock value, usually obtained
from the system via \code{gettimeofday}.  Also see \code{p4_ustimer} below.

\findex{p4_ustimer}
\begin{example}
p4_usc_time_t p4_ustimer()
\end{example}
\noindent
returns a wall-clock time value in microseconds.  The precision of this
number depends on the timer installed on the individual machine.  In
some cases the resolution may be no greater than that of \code{p4_clock()}.
For arithmetic and printing purposes, the type \code{p4_usc_time_t} is an
unsigned long integer.

\findex{p4_usrollover}
\begin{example}
p4_usc_time_t p4_usrollover()
\end{example}
\noindent
returns the timer value at which a microsecond timer ``rolls over''.
Since \code{p4_usc_time_t} is a long integer's worth of microseconds, it is
likely that the timer will roll over (become zero) during even
medium-length runs.


\node Functions for Debugging p4 Programs,Miscellaneous Functions,Functions for Timing p4 Programs,Top

\section{Functions for Debugging p4 Programs}
\cindex{debugging}

P4 has a set of routines to aid in producing a printed trace of events, both
user-defined and pre-defined in the p4 system.

\findex{p4_dprintf}
\begin{example}
VOID p4_dprintf(fmt, va_alist)
char *fmt;
va_dcl
\end{example}
\noindent
acts just like the standard \code{printf} except that the print line is
preceded by a value that identifies the process.  This value is
typically the string \code{pn_u} where \code{n} represents the
p4-assigned id and \code{u} represents the unix-id of the process on its
host.  However, there are other forms of this value.  For example, the
big master is represented as \code{bm_u}.  Also, if a process prints
before it has a p4-assigned id, then its value will be something like
\code{bm_slave_n_u} or \code{rm_slave_n_u}.  Typically, it is not
possible for a user program to print anything before being assigned an
id by p4, but the p4 system itself may use this procedure to print
messages from a particular process if it encounters problems getting the
process initialized.

\findex{p4_dprintfl}
\begin{example}
VOID p4_dprintfl(level, fmt, va_alist)
int level;
char *fmt;
va_dcl
\end{example}
\noindent
is like \code{p4_dprintf} except that the first argument is an integer
indicating the debugging level that must be in effect before this
message will print.  A level of 0 will cause the message to always print.
If you run a program with the debug level set to 5 (via command-line
arguments), then all \code{dprintfl}'s with level less than or equal to
that debug level will print. \xref{Command-Line Arguments} for how to
set the debug level at run time.

The debug level can be examined and changed by the user during execution:

\findex{p4_get_dbg_level}
\begin{example}
int p4_get_dbg_level()     
\end{example}
\noindent
returns the current debug level for this process and its cluster.

\findex{p4_set_dbg_level}
\begin{example}
VOID p4_set_dbg_level(level)     
int level;
\end{example}
\noindent
sets the current debug level for this process and its cluster.
P4 itself is liberally instrumented with \code{p4_dprintfl}'s of level
10 and above, leaving levels 0-9 for the user.  The greater the debug
level of the built-in messages, the greater understanding of p4 needed
by the user to make sense of them.  However, levels as high as 30 may
well be useful to the user trying to debug a p4 program.
Roughly speaking, the following debug levels produce messages about the
indicated events.

\begin{example}
level 10:  created process
           sent message
           received message

level 20:  creating process
           sending message
           receiving message
           process starting
           process exiting

level 30:  waiting for ack
           sending ack
           sent ack
           received ack
           queueing message for later receipt
           queued message for later receipt

level 40:  memory management
           buffer management

level 50:  reading procgroup
           other initialization message exchange

level 60:  send-receive details, especially machine-specific traces

level 70:  listener interactions:
            creating listener
            created listener
            messages from inside listener

level 80:  detailed data structures after initialization

level 90:  detailed tracing of flow thru procedures
\end{example}

For optimum performance, the test of the debug level required by these
messages can be removed at compile time by not commenting out the 
\code{#define P4_DPRINTFL} line in the \file{OPTIONS} file.
(\xref{Structure of the Distribution Directory}).

The following function is provided to deal with abnormal termination.
It can be called by any process.

\findex{p4_error}
\begin{example}
VOID p4_error(string, value)
char *string;
int value;
\end{example}
\noindent
prints \code{string} as an error message and then forcefully terminates
all co-operating processes and cleans up all shared resources.

\findex{p4_soft_errors}
\begin{example}
VOID p4_soft_errors(onoff)     
int onoff;
\end{example}
\noindent
enables/disables soft errors, returning the previous setting.  The default
is ``disabled'', which means that certain p4 functions will call
\code{p4_error} instead of returning -1.

\cindex{interrupts}
\code{p4_error} gets control on certain kinds of interrupts.  It is
automatically called for \code{SIGSEGV}, \code{SIGBUS}, and \code{SIGFPE}
interrupts, to catch user programming errors and clean up, after which it
returns interrupt handling to default mode and returns, so that the user
may obtain a dump.  It also handles \code{SIGINT} interrupts, in which case
it cleans up and exits.  Finally, it may be called directly by the user, in
which case it cleans up (other p4 processes and IPC's) and exits.

\cindex{runaway processes}
Although \code{p4_error} is supposed to get rid of all running p4
processes, it can happen that an error is bad enough that p4 processes
are left running.  A primitive aid in finding and killing these
processes is the shell script \code{kj}, which takes a string as an
argument and then kills processes containing that string as part of
their program names.  Currently it only kills processes on the machine
where it is run, but it can be run via \code{rsh} on remote machines.
There are other useful scripts (e.g. \code{killipc} and \code{killp4})
in the \file{p4/bin} directory to do such things as clean up SYSV IPC 
items that may be left when a program abnormally terminates.  P4 will 
generally cleanup these items if the abnormal termination is a type that 
p4 traps, otherwise the user must do the cleanup.  This is an unfortunate
side-effect of the way that SYSV handles things, it really should be
the OS's function to take care of this.

On many machines it is possible to attach a debugger like \code{dbx} to a
running process.  This is one way to find out where a hanging process is
stuck. 

\node Miscellaneous Functions,Fortran Interface,Functions for Debugging p4 Programs,Top

\section{Miscellaneous Functions}

In this section are found functions that do seem to fit neatly into any of the
other sections.

\findex{p4_version}
\begin{example}
char *p4_version()
\end{example}
\noindent
returns a string containing the version number of p4 being run.


\findex{p4_get_cluster_masters}
\begin{example}
VOID p4_get_cluster_masters(numids, ids)
int *numids, ids[];
\end{example}
\noindent
This procedure fills in the values of numids and ids.  It obtains the
p4-ids of all ``cluster masters'' for the program, placing them in the
ids array and placing the number of ids in numids.

\findex{p4_print_avail_buffs}
\begin{example}
VOID p4_print_avail_buffs()
\end{example}
P4 maintains an array of buffer lists of various sizes, so that it can very
rapidly allocate and deallocate buffers.  You can see the contents of the
buffer pools at any time by calling this procedure.

\cindex{buffer pool management}
\findex{p4_set_avail_buff}
\begin{example}
VOID p4_set_avail_buff(bufidx,size)
int bufidx;
int size;
\end{example}
\noindent
This procedure is used to set the size of buffers in p4's buffer pools.  The
parameter \code{bufidx} specifies a particular buffer list, and should be a
number from 0 to 7.  The \code{size} parameter specifies that buffers up to
that size will be managed by p4 in a particular list.  It is important to
maintain the buffer sizes in increasing order.  The default list of buffer
sizes is {64, 256, 1024, 4096, 16384, 65536, 262144, 1048576}.  This causes
wasted space if you send only one large message, causing the allocation of a
large buffer which is not reused.  Savings in space can be achieved by
adjusting these numbers to correspond with the message sizes of your
application.  If no large messages are sent at all, however, no space is
wasted since the large buffers will never be allocated.  If you send a message
larger that the largest size in this array, p4 will allocate the buffer, and
then free it back to the system as soon as it can.


\node Fortran Interface,Faster Startup with the Secure Server,Miscellaneous Functions,Top

\section{Fortran Interface}
\cindex{Fortran interface}

The Fortran calls to p4 procedures are analogous to their C counterparts, but
have Fortran-like names.

\begin{example}
#include "p4f.h"

p4init()
p4crpg()
p4cleanup()
p4sendr(type,dest,msg,len,rc)
p4sendrx(type,dest,msg,len,data_type,rc)
p4send(type,dest,msg,len,rc)
p4sendx(type,dest,msg,len,data_type,rc)
p4recv(type,from,buf,buflen,msglen,rc)
p4brdcst(type,data,len,rc)
p4brdcstx(type,data,len,data_type,rc)
p4probe(type,from,rc)
p4myclid()
p4nclids()
p4getclids(start,end)
p4myid()
p4clock()
p4ustimer()
p4ntotids()
p4error(str,val)
p4softerrs(new,old)
p4version()
p4flush()
p4globop(type,x,nelem,size,op,data_type,rc)
   p4intsumop()
   p4intabsmaxop()
   p4intabsminop()
   p4intmaxop()
   p4intminop()
   p4intmultop()
   p4dblsumop()
   p4dblabsmaxop()
   p4dblabsminop()
   p4dblmaxop()
   p4dblminop()
   p4dblmultop()
   p4fltsumop()
   p4fltabsmaxop()
   p4fltabsminop()
   p4fltmaxop()
   p4fltminop()
   p4fltmultop()
\end{example}

The \code{data_type} parameter in the above operations shoudl be one of
\code{P4INT}, \code{P4LNG}, \code{P4FLT}, or \code{P4DBL}.
These symbolic constants are defined in the include file \file{p4f.h}.

\node Faster Startup with the Secure Server,Utilities for Managing a p4 Session,Fortran Interface,Top

\section{Faster Startup with the Secure Server}
\cindex{faster startup}
\cindex{servers}
\cindex{permissions}
\cindex{rhosts}

P4 processes on remote machines are ordinarily created by \code{rsh}.
For this to work, the user must have permission to create processes on
that machine.  This permission is normally granted either globally by
the system administrator, or locally by the use of \file{.rhosts} files.
(See the normal unix man pages under \code{rhosts}).

Since \code{rsh} is relatively slow, p4 provides a way to get things
started faster.  This is accomplished by running the program
\code{serv_p4} in the background on the remote machine.  When p4 is
creating processes, it will automatically check for the existence of
this server and use it if it is running.  Remote processes typically
start much faster when the server is running.  A disadvantage of using
the server is that output from \code{p4_dprintf} is lost unless directed
by a command-line argument to a file.  (\xref{Command-Line Arguments}.)
When p4 uses \code{rsh}, the remote process's \code{stdout} is sent
back to the \code{stdout} of the parent (the p4 master process).
We have not yet tested this server on all of the machines that we support.
Thus far, we have tested it somewhat on the SYMMETRY, SUN, DEC500, and
SGI.  We believe that it will work on many other machines, but have not
yet verified it on all machines.

An invocation of a set of servers is (currently) associated with a specific
port number.  This way multiple users can each be running multiple server
networks without mutual interference, provided each network of servers is
started with a different port number. 

To start the secrue server on a machine one can do

\begin{example}
serv_p4 -d -p <num>
\end{example}

\noindent
where \code{<num>} is a port number to be associated with a network of
servers.  If the \code{-p} option is omitted, the server will pick an unused
port number and report

\begin{example}
Listening on <num>.
\end{example}

\noindent
Then p4 programs to use this network should be started with 

\begin{example}
-ssport <num>
\end{example}

The p4 application must also be listed in the user's \file{.p4apps} file in
his home directory.  This file should be readable only by the user, and should
contain the full path names of programs that the user wishes to be startable
by the p4 server.

When a p4 master process tries to start a slave process on a remote machine,
it will first attempt to do it via the server.  If it cannot do so for any
reason (no server running, port number mismatch, or program not found in
\file{.p4apps} file), then it tries to do so with the remote shell command.

Note that the server is used only to start processes; it plays no role in a p4
computation once the slave processes have been initiated.  Rather, a temporary
process, called the {\em listener}, is spawned to manage connection requests
that occur during the execution of a p4 program.  Neither the server nor the
listener consumes any significant amount of CPU time.

There is further discussion of installation options for the servers in
the \file{README} file in the \file{p4/servers} subdirectory.

\node Utilities for Managing a p4 Session,Creating Logfiles for Upshot,Faster Startup with the Secure Server,Top

\section{Utilities for Managing a p4 Session}
\cindex{utilities}
\cindex{servers}

A number of useful utilities can be found in the \file{bin} subdirectory.
These can be used to start and stop server processes based on the contents of
a file of machines one regularly uses, to kill runaway p4 processes in the
unlikely case that they cannot or do not terminate automatically when one
processes ends abnormally or interrupted from the keyboard, and to merge
logfiles created for the use of \code{upshot} \xref{Creating Logfiles for
Upshot}. 

\node Creating Logfiles for Upshot,Machine-Specific Notes,Utilities for Managing a p4 Session,Top

\section{Creating Logfiles for Upshot}
\cindex{tracefiles}
\cindex{upshot}
\cindex{logging}
\cindex{timestamps}

P4 is distributed with a set of routines for creating logfiles (see
\file{README} in the \file{p4/alog} directory.  The resulting logfiles
can be examined by \code{upshot}, distributed separately.  For details
about \code{upshot}, see \cite{herrarte-lusk:upshot}.  

The \file{p4/alog} directory contains a package (ALOG) for creating logs of
time-stamped events, that is of general utility, outside of p4.  The
timestamps are obtained from various microsecond-level resolution timers on
various machines.  The portable microsecond timing package is contained in the
\file{usc} subdirectory.  It is used by the ALOG package as well as by the
\code{p4_ustimer} function in p4.  Similarly, the ALOG package can be used
independently of p4 and \code{upshot}. Its logfiles were designed to be read
and displayed by upshot, but other display packages can be used as well.

\begin{menu}
* User-Specified Events::
* Examining Log Files with Upshot::
* Automatic Logging of p4 Events::
\end{menu}

\node User-Specified Events,Examining Log Files with Upshot,	,Creating Logfiles for Upshot

\subsection{User-Specified Events}
\cindex{user events}

The ALOG package consists of a set of macros that can be used to instrument a
C program and a set of functions that can be used to instrument a Fortran
program.  We will focus here primarily on the use of the C interface, which
contains more functionality.

The macros that can be used to instrument a program are as follows (from the 
file \file{README_ALOG} in the \file{alog} directory):

\findex{ALOG_SETUP}
\begin{example}
ALOG_SETUP(pid,flag):	
  pid  -  (integer) process id of callee
  flag -  (integer) either ALOG_WRAP or ALOG_TRUNCATE  
\end{example}
\noindent
This macro initializes the tracing area for a slave process and must be called
once before any event is logged.  If the value of \code{flag} is set to
\code{ALOG_WRAP}, then in the event of no more space for logging events the
system will only report the latest n events.  If \code{flag} is set to
\code{ALOG_TRUNCATE} the system will stop logging events as soon as there is
no more memory for the events to be logged.

\findex{ALOG_MASTER}
\begin{example}
ALOG_MASTER(pid,flag):
  pid  -  (integer) process id of the callee
  flag -  (integer) either 0 or 1  (see above)
\end{example}
\noindent This macro has the same effect over its parameters as
\code{ALOG_SETUP} with the difference that this macro should be referenced by
the master process only.

\findex{ALOG_DEFINE}
\begin{example}
ALOG_DEFINE(event,strdef,format):
  event  - (integer) id of event being defined
  strdef - (string) description of 'event'
  format - (string) control string in "printf" format to
\end{example}
\noindent
This macro puts an event definition code into the logfile.

\findex{ALOG_LOG}
\begin{example}
ALOG_LOG(pid,event,intdata,strdata):
  pid     - (integer) process id of callee
  event   - (integer) event id to be logged
  intdata - (integer) any integer data for this event
  strdata - (string) any string data (can be the null string)
\end{example}
\noindent
This macro provides the event logging service.

\findex{ALOG_OUTPUT}
\begin{example}
ALOG_OUTPUT
  no parameters  
\end{example}
\noindent
This macro dumps the events logged into a log file with the name
\file{alogfile.pxx} where \code{xx} is the logical PID of the callee process.
The log file is created in the current directory unless specified otherwise
through the macro ALOG_SETDIR.

\findex{ALOG_SETDIR}
\begin{example}
ALOG_SETDIR(dir)
  dir - (string) directory where log file is created
\end{example}
\noindent
This macro sets the output directory for the log file.  The default directory
for the creation of the log file is the current directory of the process.  If
used, then this macro MUST be invoked before \code{ALOG_MASTER/ALOG_SETUP}.

\findex{ALOG_STATUS}
\begin{example}
ALOG_STATUS(status):
  status - (integer) either ALOG_ON or ALOG_OFF
\end{example}
\noindent

This macro controls the logging status of \code{ALOG} as follows.  Setting
\code{status} to \code{ALOG_ON} enables logging until it is turned off.
Setting \code{status} to \code{ALOG_OFF} disables logging until it is turned
on again.  Logging is enabled at the outset by default.


\findex{ALOG_ENABLE}
\begin{example}
ALOG_ENABLE
  no parameters
\end{example}
\noindent

This macro enables event logging; same as calling \code{ALOG_STATUS(ALOG_ON)}.

\findex{ALOG_DISABLE}
\begin{example}
ALOG_DISABLE
  no parameters
\end{example}
\noindent
This macro disables event logging; same as calling
\code{ALOG_STATUS(ALOG_OFF)}.

The sample program \file{gridlog.shmem.c} in the \file{monitors} subdirectory
contains an example of a program instrumented with ALOG statements.  The macro
definitions for ALOG are included when you include \code{#include "p4"} in
your program.  If the line \code{#define ALOG_TRACE} is not included before
the \code{#include "p4"}, these macros will generate no code.  Thus it is
easy to effectively de-instrument the code by recompiling, and there is no
need to protect each ALOG statement with an \code{#ifdef}.
 
When an ALOG-instrumented program is run, it will produce one logfile for each
process.  The files will be named \file{alogfile.p0}, \file{alogfile.p1},
\ldots.  These files need to be merged into a single file with the events
sored by timestamp.  This is accomplished with the program \file{mergelogs},
found in the \file{bin} subdirectory.  To merge the logfiles, do 

\begin{example}
mergelogs alogfile.p* > myprog.log
rm alogfile.p*
\end{example}

The resulting logfile can be examined by upshot or some other logfile
examination facility.  See \cite{herrarte-lusk:upshot} for details of the
logfile format.

On networks of workstations and some distributed memory machisnes, the
microsecond timers on the various processors are synchronized.  To produce a
usable merged logfile, the \file{adjlogs} program, also found in the
\file{bin} directory, can be used to adjust the timestamps for offset and
drift before they are merged.  For this to work, synchronization events must
be placed in the logfiles by an \code{ALOG_LOG} statement.  The event type
is then passed to \code{adjlogs}, which aligns the timestamps, based on the
timestamps of the synchonization events.  The call to \code{adjlogs} looks
like this, where \code{<n>} is the type of the synchronization event.

\begin{example}
adjlogs -e <n>
\end{example}

Both \code{mergelogs} and \code{adjlogs} are less portable than the other p4
code;  you might want to run them on a workstation such as a Sun.

\node Examining Log Files with Upshot,Automatic Logging of p4 Events,User-Specified Events,Creating Logfiles for Upshot

\subsection{Examining Log Files with Upshot}
\cindex{upshot}
\cindex{statefile}

\code{Upshot} is not part of the p4 distribution, but can be obtained from the
same anonymous \code{ftp} location as p4.  Take the file \file{upshot.tar.Z}
from the directory \file{pub/p4} on \code{info.mcs.anl.gov}.  The distribution
contains all necessary documentation on how to install and run \code{upshot}.
It is an X-window program that runs on most workstations.  There is no need
for a parallel macchine to be involved, once the log files have been obtained.

\code{Upshot} produces the most interesting displays when certain events
(not necessarily all) are defined to be the entry and exit events for certain
{\it states} and then colors are associated with the states.  This association
is reflected in a {\it statefile} with a format like the following:

\begin{example}
1 1 2 red   asking
2 3 4 blue  working
3 5 6 green updating
\end{example}

This statefile describes three states.  State 1 is defined to be between
events 1 and 2.  \code{Upshot} will color it red and label it ``asking''.

\node Automatic Logging of p4 Events,	,Examining Log Files with Upshot,Creating Logfiles for Upshot

\subsection{Automatic Logging of p4 Events}
\cindex{automatic logging}

We have found that the most useful events to log and study are those
identified by the user and specified in his program.  That way he can control
the number of events to be logged and the grain size of the states that are
represented.  

In some cases, however, one wants to study the details of the internal
operation of a p4 application, or get some idea of the behavior on one's
program without going to the trouble of instrumenting it himself.  To this
end, p4 itself is instrumented with ALOG statements, although by default they
are inactive.  To get automatic logging of p4 events (including sending and
receiving of each message) one needs first to link to a version of the p4
library that has been compiled with the line \code{#define ALOG_TRACE}
uncommented out in the \file{OPTIONS} file, and secondly, to run with
\code{-p4log} on the command line.

\node Machine-Specific Notes,Some Common Problems and their Solutions,Creating Logfiles for Upshot,Top

\section{Machine-Specific Notes}

\begin{example}
SUN
    (1)  P4 can be installed on this machine with or without SYSV IPC.

HP
    (1)  P4 can be installed on this machine with or without SYSV IPC.
    (2)  Fortran not tested (not avail on our test machine).

DEC5000
    (1)  P4 can be installed on this machine with or without SYSV IPC.

RS6000
    (1)  P4 can be installed on this machine with or without SYSV IPC.

IBM3090
    (1)  P4 can be installed on this machine with or without SYSV IPC.
    (2)  Fortran not supported due to absence of iargc/getarg.
    (3)  There are multiply defined macros in include/rpc/rpc.h.  IBM
         is fixing this in a later OS release.  Meanwhile make your own
         copy and the fix the problem yourself.
    
TITAN
    (1)  P4 can be installed on this machine with or without SYSV IPC.
    (2)  Fortran not supported due to problems with getting args.

SGI
    (1)  P4 can be installed on this machine with or without SYSV IPC.

NEXT
    (1)  Fortran not supported due to absence of iargc/getarg.

FX2800/FX2800_SWITCH
    (1)  Alliant's switch code not yet ensuring messages 
         remain ordered.  p4 currently discovers the switch port for
         the machine it is running on by invoking the internal
         procedure getswport.  This procedure must be customized to
         the installation.
      
FX8
    (1)  You might need to add MFLAGS = -i to the Makefile

KSR
    (1)  The latest version of the OS produces a link-time error for
         Fortran programs.

IPSC860
    (1)  the script ``runcube'' (in the messages directory) may 
         be useful
	 
DELTA
    (1)  the script ``rundelta'' (in the messages directory) may 
         be useful

BALANCE
    (1)  Fortran not supported.

SYMMETRY/SYMMETRY_PTX
    (1)  -Z compiler option may be used to control shmalloc/malloc split
    (2)  shared memory message passing not supported in Fortran

TC_2000/TC_2000_TCMP
    (1)  TCMP port not yet complete.
    (2)  For shared-memory execution, one must use \code{cluster ...} to
         obtain a private cluster for execution
\end{example}

\node Some Common Problems and their Solutions,Concept Index,Machine-Specific Notes,Top

\section{Some Common Problems and their Solutions}
\cindex{problems}
\cindex{permission denied}

Our attempt with this manual has been to prevent you from having difficulties.
Experience shows that certain common progblems recur, however.  In this
section we hope to address some of these problems.

\begin{description}
\item[``Permission Denied.''] p4 slave processes are started by forks (for
  slaves in the same shared-memory cluster), by the server, or by the remote
  shell command.  If the server is running on the target machine then that
  must be configured to allow remote processes to be started.  To test whether
  this is your problem, try
\begin{example}
    rsh target.machine date
\end{example}
  If you still get the ``Permission denied.'' message, then the problem has
  nothing to do with p4.  See \code{hosts.equiv} or \code{.rhosts} in the
  system man pages.
\item[``More processes than message queues'']  Under the default configuration
  of p4, uniprocessors, such as most workstations, cannot have multiple
  process sharing memory.  Thus your procgroup file for a workstation network
  should always look like
\noindent
\begin{example}
    local 0
    machine1 1 pathname
    machine2 1 pathname
    machine3 1 pathname
       .
       .
\end{example}
  The ``local'' means ``only the master on the startup machine; no local
  slaves sharing memory''.

\cindex{IPC}
  It is possible, at some cost in message-passing efficiency, to have a
  cluster of processes sharing memory on a workstation, but in this case p4
  must have been installed with the \code{SYSV_IPC} option set in the
  \file{OPTIONS} file.  The cost is that a process waiting for a message must
  spin between checking for a message arriving on a socket and a message
  arriving through shared memory. 

\item[``p4\_error: local is not first entry in procgroup'']  the first line of
  the procgroup file must be the ``local'' entry, specifying the number of
  slaves that will be run on the master machine in addition to the master
  process. 

\item[``gethostbyname failed 100 times''] Check for an invalid machine name in
  the procgroup file.
\item[``pgm\_path\_name: Command not found'']  P4 tried to start the program
  with the given name on a remote machine and the program did not exist.
  Verify the full path name of the program.
\item[program hangs]  You may have failed to initialize the \code{type} and
  \code{from} fields before a \code{p4_recv}.  You might have used
  \code{p4_sendr} between two processes at the same time, which will deadlock
  if you think about it, or even if you don't.  Use \code{p4_send} instead.
\item[program hangs or has bad data in received message]  You might have
  failed to set the pointer to the incoming buffer to NULL, or to have
  specifically allocated a buffer with \code{p4_msg_alloc}, before a
  \code{p4_recv}.
\item[program ignores command-line arguments]  You might have passed
  \code{argc} instead of \code{&argc} to \code{p4_initenv}.
\item[program runs out of memory]  You may need to call \code{p4_msg_free}
  after each \code{p4_recv}, or reuse buffers by pre-allocating them.


\end{description}

\twocolumn
\node Concept Index,Function Index,Some Common Problems and their Solutions,Top

\section{Concept Index}
\printindex{cp}


\node Function Index,	,Concept Index,Top

\section{Function Index}

\printindex{fn}

\onecolumn

\begin{tex}
\bibliography{p4}
\end{tex}

\end{document}

SHAR_EOF
if test -f 'p4.1'
then
	echo shar: over-writing existing file "'p4.1'"
fi
cat << \SHAR_EOF > 'p4.1'
.TH p4 1 "7 November 1992"

.SH NAME 
p4 - portable programs for parallel processors - release 1.2 

.SH DESCRIPTION
p4 is a package of procedures that support development of parallel programs
that are portable across a wide variety of architectures.  For the
shared-memory model of parallel computation, p4 provides a set of primitives
from which monitors can be constructed, as well as a set of useful monitors.
For the distributed-memory model, p4 provides typed send and receive
operations, and creation of processes according to a text file describing
files to be executed on by created processes and the machines to execute them
on.  The complete package consists of several data types and procedures that 
are briefly outlined in the SYNOPSIS section below.

There is also a fortran interface to p4 (p4f).

.SH SYNOPSIS
    #include "p4.h"

    a set of abstract data types sharable among processes:

             p4_lock_t (locks)
             p4_monitor_t (general monitors)
             p4_barrier_monitor_t
             p4_getsub_monitor_t
             p4_askfor_monitor_t


    a set of procedures for process management:

             p4_create()
             p4_create_procgroup()
             p4_wait_for_end()
             p4_get_my_id()
             p4_get_my_cluster_id()
             p4_get_cluster_ids()
             p4_am_i_cluster_master()
             p4_num_total_ids()
             p4_num_total_slaves()
             p4_num_cluster_ids()



    a set of procedures for manipulating shared data items
         and coordinating the activities of processes that use them:

             p4_shmalloc()
             p4_shfree()
             p4_malloc()
             p4_free()
             p4_lock_init()
             p4_lock()
             p4_unlock()
             p4_barrier()
             p4_barrier_init()
             p4_getsub()
             p4_getsub_init()
             p4_askfor()
             p4_askfor_init()
             p4_update()
             p4_moninit()
             p4_menter()
             p4_mexit()
             p4_mcontinue()
             p4_mdelay()
             p4_probend()
             p4_progend()


    a set of procedures for sending/receiving messages:

             p4_send()
             p4_sendr()
             p4_sendx()
             p4_sendrx()
             p4_sendb()
             p4_sendbr()
             p4_sendbx()
             p4_sendbrx()
             p4_recv()
             p4_msg_free()
             p4_msg_alloc()
             p4_messages_available()


    additional procedures:

             p4_version()
             p4_initenv()
             p4_clock()
             p4_dprintf()
             p4_dprintfl()
             p4_get_dbg_level()
             p4_set_dbg_level()
             p4_print_avail_buffs()
             p4_set_avail_buff()
             p4_broadcast()
             p4_broadcastx()
             p4_global_barrier()
             p4_global_op()
               p4_int_absmax_op()
               p4_int_absmin_op()
               p4_int_max_op()
               p4_int_min_op()
               p4_int_mult_op()
               p4_int_sum_op()
               p4_dbl_absmax_op()
               p4_dbl_absmin_op()
               p4_dbl_max_op()
               p4_dbl_min_op()
               p4_dbl_mult_op()
               p4_dbl_sum_op()
	       p4_flt_absmax_op()
	       p4_flt_absmin_op()
	       p4_flt_max_op()
	       p4_flt_min_op()
	       p4_flt_mult_op()
	       p4_flt_sum_op()



.SH OPTIONS
Several command line options are available to any program linked to 
the p4 package.
.TP 
-help   	get this information
.TP 
-pg     	procgroup_file
.TP 
-dbg    	set debug level
.TP 
-rdbg   	set remote debug level
.TP 
-gm     	set globmemsize
.TP 
-dmn    	domainname
.TP 
-out    	file 	output file for master
.TP 
-rout   	file 	output file prefix for remote masters
.TP 
-ssport         port# private port number for secure server
.TP 
-p4log          enable internal p4 logging by alog
.TP 
-p4version   	print the current p4 version number

.SH FILES
p4.h, libp4.a - see your systems administrator for the installed location

.SH DIAGNOSTICS
p4 generates a wide variety of diagnostic messages.  See the user
manual for more details.

.SH EXAMPLE EXECUTION
To invoke a program linked with p4 using a procgroup file:
.RS
.nf
myprogram -pg procgroup_file_name
.fi
.RE

.SH SEE ALSO
.BR socket, etc.

.SH BUGS
There are no
.I known 
bugs at this time, but stay tuned.
SHAR_EOF
if test -f 'p4f.1'
then
	echo shar: over-writing existing file "'p4f.1'"
fi
cat << \SHAR_EOF > 'p4f.1'
.TH p4f 1 "1 January 1992"

.SH NAME 
p4f - the fortran interface to p4

.SH DESCRIPTION
p4f is a package of procedures that support development of parallel programs
that are portable across a wide variety of architectures.
It is actually a fortran interface to the p4 parallel programming package.
the synopsis section below defines the set of fortran routines that may 
be invoked from a fortran program.

.SH SYNOPSIS
    #include "p4f.h"

    p4init()
    p4crpg()
    p4cleanup()
    p4sendr(type,dest,msg,len,rc)
    p4sendrx(type,dest,msg,len,data_type,rc)
    p4send(type,dest,msg,len,rc)
    p4sendx(type,dest,msg,len,data_type,rc)
    p4recv(type,from,buf,buflen,msglen,rc)
    p4brdcst(type,data,len,rc)
    p4brdcstx(type,data,len,data_type,rc)
    p4probe(type,from,rc)
    p4myclid()
    p4nclids()
    p4getclids(start,end)
    p4globarr(type)
    p4getclmasts(numids,ids)
    p4myid()
    p4clock()
    p4ustimer()
    p4ntotids()
    p4nslaves()
    p4error(str,val)
    p4avlbufs()
    p4softerrs(new,old)
    p4version()
    p4flush()
    p4globop(type,x,nelem,size,op,data_type,rc)
       p4_int_sum_op()
       p4_int_absmax_op()
       p4_int_absmin_op()
       p4_int_max_op()
       p4_int_min_op()
       p4_int_mult_op()
       p4_dbl_sum_op()
       p4_dbl_absmax_op()
       p4_dbl_absmin_op()
       p4_dbl_max_op()
       p4_dbl_min_op()
       p4_dbl_mult_op()
       p4_flt_sum_op()
       p4_flt_absmax_op()
       p4_flt_absmin_op()
       p4_flt_max_op()
       p4_flt_min_op()
       p4_flt_mult_op()



.SH OPTIONS
Several command line options are available to any program linked to 
the p4 package.
.TP 
-pg     	procgroup_file
.TP 
-dbg    	set debug level
.TP 
-rdbg   	set remote debug level
.TP 
-gm     	set globmemsize
.TP 
-dmn    	domainname
.TP 
-out    	file 	output file for master
.TP 
-rout   	file 	output file prefix for remote masters
.TP 
-p4version   	print the current p4 version number
.TP 
-help   	get this information

.SH FILES
p4f.h, libp4.a libp4f.a - see your systems administrator for the installed location

.SH DIAGNOSTICS
p4 generates a wide variety of diagnostic messages.  See the user
manual for more details.

.SH EXAMPLE EXECUTION
To invoke a program linked with p4 using a procgroup file:
.RS
.nf
myprogram -pg procgroup_file_name
.fi
.RE

.SH SEE ALSO
.BR socket, etc.

.SH BUGS
There are no
.I known 
bugs at this time, but stay tuned.
SHAR_EOF
if test -f 'latexinfo.sty'
then
	echo shar: over-writing existing file "'latexinfo.sty'"
fi
cat << \SHAR_EOF > 'latexinfo.sty'
%% TeX macros to handle latexinfo files
%% \include{ 5 page GNU copyleft here }
%   Copyright (C) 1985, 1986 Richard M. Stallman
%   Copyleft  (0) 1988, 1989 Michael E. Clarkson

\def\latexinfoversion{1.2}
\message{Loading Latexinfo package [Version \latexinfoversion]:}
\message{}

\setcounter{tocdepth}{2}
\setcounter{secnumdepth}{3}

\def\oddfoot#1{\def\@oddfoot{#1}}
\def\evenfoot#1{\def\@evenfoot{#1}}

% Save some parts of plain tex whose names we will redefine.

\let\ptexlbrace=\{
\let\ptexrbrace=\}
\let\ptexdot=\.
\let\ptexstar=\*
\let\ptexend=\end
\let\ptexbullet=\bullet
\let\ptexb=\b
\let\ptexc=\c
\let\ptexi=\i
\let\ptext=\t
\let\ptexl=\l
\let\ptexL=\L
\def\pline{\hbox to\hsize}
\def\pleftline#1{\pline{#1\hss}}
\def\prightline#1{\pline{\hss#1}}

\def\centerline#1{\pline{\hss#1\hss}}

\def\tie{\penalty 10000\ }     % Save plain tex definition of ~.

\let\normal=\normalsize

\message{Basics,}
\chardef\other=12

\hyphenation{ap-pen-dix}
\hyphenation{mini-buf-fer mini-buf-fers}

% Parse an argument, then pass it to #1.
% The argument can be delimited with [...] or with "..." or braces
% or it can be a whole line.
% #1 should be a macro which expects
% an ordinary undelimited TeX argument.

\def\parsearg #1{\let\next=#1\begingroup\obeylines\futurelet\temp\parseargx}

\def\parseargx{%
\ifx \obeyedspace\temp \aftergroup\parseargdiscardspace \else%
\aftergroup \parseargline %
\fi \endgroup}

{\obeyspaces %
\gdef\parseargdiscardspace {\begingroup\obeylines\futurelet\temp\parseargx}}

\gdef\obeyedspace{\ }

\def\parseargline{\begingroup \obeylines \parsearglinex}
{\obeylines %
\gdef\parsearglinex #1^^M{\endgroup \next {#1}}}

\def\flushcr{\ifx\par\lisppar \def\next##1{}\else \let\next=\relax \fi \next}

% Single-spacing is done by various environments.

\newskip\singlespaceskip
\singlespaceskip = \baselineskip
\def\singlespace{%
{\advance \baselineskip by -\singlespaceskip
\kern \baselineskip}%
\baselineskip=\singlespaceskip
}

\def\arrow{$\rightarrow$}

% Used to generate quoted braces.

\def\mylbrace {{\tt \char '173}}
\def\myrbrace {{\tt \char '175}}
\let\{=\mylbrace
\let\}=\myrbrace

% \: forces normal size whitespace following.
\def\:{\spacefactor=1000 }

% \* forces a line break.
\def\*{\hfil\break}

% \. is an end-of-sentence period.
\def\.{.\spacefactor=3000 }

% \w prevents a word break
\def\w#1{\hbox{#1}}

% \br   forces paragraph break

\let\br = \par

% \dots{}  output some dots

\def\dots{$\ldots$}

% \page    forces the start of a new page
%\def\page{\clearpage}

% \exdent text....
% outputs text on separate line in roman font, starting at standard page margin

\def\exdent{\errmessage{@exdent in filled text}}
  % \lisp, etc, define \exdent locally from \internalexdent

{\obeyspaces
\gdef\internalexdent{\parsearg\exdentzzz}}

\def\exdentzzz #1{{\advance \leftskip by -\lispnarrowing
\advance \hsize by -\leftskip
\advance \hsize by -\rightskip
\leftline{{\rm#1}}}}

% \sp n   outputs n lines of vertical space
\def\sp#1{\par\vskip #1\baselineskip\relax}

% \comment ...line which is ignored...
% \c is the same as @comment
% \begin{ignore} ... \end{ignore}  is another way to write a comment
\def\comment{\parsearg \commentxxx}

\def\commentxxx #1{}

\let\c=\comment

% Some texinfo constructs that are trivial in tex
\newenvironment{iftex}{}{}

\catcode`@=11

\message{menus,}
{\catcode`\^^M=13 \gdef\@gobblecr{\@ifnextchar
{\@gobble}{\ignorespaces}}}

\chardef\active=13

{\catcode`\ =\active\gdef\@vobeyspaces{\catcode`\ \active \let \@xobeysp}}
 
\begingroup \catcode `|=0 \catcode `[= 1
\catcode`]=2 \catcode `\{=12 \catcode `\}=12
\catcode`\\=12 
|gdef|@xmenu#1\end{menu}[|end[menu]]
|gdef|@xifinfo#1\end{ifinfo}[|end[ifinfo]]
|gdef|@xignore#1\end{ignore}[|end[ignore]]
|endgroup

\def\@menu{\begingroup
\@tempswafalse 
\def\par{\if@tempswa\hbox{}\fi\@tempswatrue\@@par}
\obeylines \let\do\@makeother \dospecials}

\def\menu{\@menu\@xmenu }
\let\endmenu=\endgroup
\def\ifinfo{\@menu\@xifinfo }
\let\endifinfo=\endgroup
\def\ignore{\@menu\@xignore }
\let\endignore=\endgroup

\def\node{\parsearg\nodezzz}
\def\nodezzz#1{\nodexxx [#1,]}
\def\nodexxx[#1,#2]{\gdef\lastnode{#1}}
%\def\node#1#2#3#4{\gdef\lastnode{#1}}
\let\lastnode=\relax


\let\refill=\relax

\let\setfilename=\comment

\def\inforef #1{\inforefzzz #1,,,,**}
\def\inforefzzz #1,#2,#3,#4**{See Info file \file{\losespace#3{}}, node `\losespace#1{}'}
\def\losespace #1{#1}

\message{fonts,}

% Font-change commands.

\def\li{\sf}

%% Add scribe-like font environments, plus \l for inline lisp (usually sans
%% serif) and \ii for TeX italic

\def\i#1{{\sl #1}}
\let\var=\i
\let\dfn=\i
\let\emph=\i

\def\b#1{{\bf #1}}
\let\strong=\b

\def\t#1{{\tt \rawbackslash #1}\null}
\let\ttfont=\t
\let\kbd=\t
\let\code=\t
\def\samp#1{`{\tt \rawbackslash #1}'\null}
\def\key#1{{\tt \uppercase{#1}}\null}
\def\ctrl#1{$\uparrow${\tt #1}}

\let\file=\samp

\def\l#1{{\li #1}\null}

\def\r#1{{\rm #1}}
\def\s#1{{\sc #1}}
\def\ii#1{{\it #1}}

\def\bullet{$\ptexbullet$}
\def\labelitemi{$\ptexbullet$}

\def\minus{$-$}

% Make altmode in file print out right

\catcode `\^^[=\active \def^^[{$\diamondsuit$}

\message{indexing,}
% Index generation facilities

% Define the texinfo user-accessible indexing commands for backwrds 
% compatability.

\def\findex{\fnindex}
\def\kindex{\kyindex}
\def\cindex{\cpindex}
\def\vindex{\vrindex}
\def\tindex{\tpindex}
\def\pindex{\pgindex}
\def\cindex{\cpindex}


% \newindex {foo} defines an index named foo.
% It automatically defines \fooindex such that
% \fooindex ...rest of line... puts an entry in the index foo.
% It also defines \fooindfile to be the number of the output channel for
% the file that	accumulates this index.  The file's extension is foo.
% The name of an index should be no more than 2 characters long
% for the sake of vms.

\def\newindex#1{\csname newindex#1\endcsname}

% Yes I know this could be done shorter, but safer??

\def\newindexcp{\if@filesw \newwrite\@cpindexfile
  \immediate\openout\@cpindexfile=\jobname.cp
  \def\cpindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@cpindexfile}%
  \def\cpsubindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrsubindex\@cpindexfile}%
  \def\cpindexbold{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindexbold\@cpindexfile}
   \typeout{Writing concepts index file \jobname.cp }%
   \fi}

\def\cpindex{\@bsphack\begingroup \@sanitize\@index}
\def\cpsubindex{\@bsphack\begingroup \@sanitize\@index}
\def\cpindexbold{\@bsphack\begingroup \@sanitize\@index}

\def\@wrindex#1#2{\let\thepage\relax
   \xdef\@gtempa{\write#1{\string \entry{#2}{\thepage}{#2}}}\endgroup\@gtempa
   \if@nobreak \ifvmode\nobreak\fi\fi\@esphack}

\def\@wrindexbold#1#2{\let\thepage\relax \let\bf\relax
   \xdef\@gtempa{\write#1{\string \entry{#2}{\b{\thepage}}{#2}}}\endgroup\@gtempa
   \if@nobreak \ifvmode\nobreak\fi\fi\@esphack}

\def\@wrsubindex#1#2#3{\let\thepage\relax
	\xdef\@gtempa{\write#1{\string \entry{#2 #3}{\thepage}{#2}{#3}}}\endgroup%
	\@gtempa
   \if@nobreak \ifvmode\nobreak\fi\fi\@esphack}

\def\@index#1{\endgroup\@esphack}

\def\pgindex{\@bsphack\begingroup \@sanitize\@index}
\def\newindexpg{\if@filesw \newwrite\@pgindexfile
  \immediate\openout\@pgindexfile=\jobname.pg
  \def\pgindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@pgindexfile}\typeout{Writing index file \jobname.pg }\fi}

\def\newindexvr{\if@filesw \newwrite\@vrindexfile
  \immediate\openout\@vrindexfile=\jobname.vr
  \def\vrindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@vrindexfile}
  \def\vrindexbold{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindexbold\@vrindexfile}
	\typeout{Writing variable index file \jobname.vr }\fi}

\def\vrindex{\@bsphack\begingroup \@sanitize\@index}
\def\vrindexbold{\@bsphack\begingroup \@sanitize\@index}

\def\newindextp{\if@filesw \newwrite\@tpindexfile
  \immediate\openout\@tpindexfile=\jobname.tp
  \def\tpindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@tpindexfile}
	\typeout{Writing datatype index file \jobname.tp }\fi}

\def\tpindex{\@bsphack\begingroup \@sanitize\@index}

\def\newindexky{\if@filesw \newwrite\@kyindexfile
  \immediate\openout\@kyindexfile=\jobname.ky
  \def\kyindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@kyindexfile}\typeout{Writing key index file \jobname.ky }\fi}

\def\kyindex{\@bsphack\begingroup \@sanitize\@index}

\def\newindexfn{\if@filesw \newwrite\@fnindexfile
  \immediate\openout\@fnindexfile=\jobname.fn
  \def\fnindex{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindex\@fnindexfile}
  \def\fnindexbold{\@bsphack\begingroup
             \def\protect####1{\string####1\space}\@sanitize
             \@wrindexbold\@fnindexfile}
	\typeout{Writing function index file \jobname.fn }\fi}

\def\fnindex{\@bsphack\begingroup \@sanitize\@index}


% \synindex{foo}{bar}    makes index foo feed into index bar.
\def\synindex#1#2{\expandafter\xdef \csname @#1index\endcsname{%
	\noexpand\csname @#2index\endcsname}%
\expandafter \chardef \csname @#1indfile\endcsname=\csname @#2indfile\endcsname}


% Define the macros used in formatting output of the sorted index material.
% This is what you call to cause a particular index to get printed.
% Write
% \section*{Function Index}
% \printindex{fn}

\newskip\secondaryindent
\secondaryindent=0.5cm

%\catcode`\%=\other\catcode`\&=\other\catcode`\#=\other
%\catcode`\@=\other\catcode`\$=\other\catcode`\_=\other
%\catcode`\~=\other\catcode`\^=\other
%\tex %
%\catcode`\%=\other\catcode`\&=\other\catcode`\#=\other
%\catcode`\@=\other\catcode`\$=\other\catcode`\_=\other
%\catcode`\~=\other\catcode`\^=\other

\def\printindex#1{
\def\indexbackslash{\rawbackslashxx}
\tolerance=9500 \advance\baselineskip -1pt
\openin 1 \jobname.#1s
\ifeof 1 \else \closein 1 \input \jobname.#1s
\fi
}

% These macros are used by the sorted index file itself.
% Change them to control the appearance of the index.

\def\Dotsbox{\hbox to 1em{\hss.\hss}} % Used by index macros

\def\initial#1{%
{\bigbreak\pline{\large\bf#1\hfill}\kern 2pt\penalty3000}}

\def\entry #1#2{
{\parfillskip=0in \parskip=0in \parindent=0in \small
\hangindent=1in \hangafter=1%
\noindent\hbox{#1}\leaders\Dotsbox\hskip 0pt plus 1filll\relax #2\par
}}

\def\primary #1{\pline{\small#1\hfil}}

\def\secondary #1#2{
{\parfillskip=0in \parskip=0in \small
\hangindent =1in \hangafter=1
\noindent\hskip\secondaryindent\hbox{#1}\leaders\Dotsbox\hskip 0pt plus 1filll#2\par
}}

%%% Args are the skip and penalty (usually negative)
\def\dobreak#1#2{\par\ifdim\lastskip<#1\removelastskip\penalty#2\vskip#1\fi}

\message{environments,}

% \begin{tex} ... \end{tex}    escapes into raw Tex temporarily.

\def\tex{\begingroup
\catcode `\\=0 \catcode `\{=1 \catcode `\}=2
\catcode `\$=3 \catcode `\&=4 \catcode `\#=6
\catcode `\^=7 \catcode `\_=8 \catcode `\~=13 \let~=\tie
\catcode `\%=14
\catcode`\"=12
\catcode`\|=12
\catcode`\<=12
\catcode`\>=12

\let\{=\ptexlbrace
\let\}=\ptexrbrace
\let\.=\ptexdot
\let\*=\ptexstar
\let\bullet=\ptexbullet
\let\b=\ptexb \let\c=\ptexc \let\i=\ptexi \let\t=\ptext \let\l=\ptexl
\let\L=\ptexL

\let\endtex=\endgroup}

% Define \lisp ... \endlisp.
% \lisp does a \begingroup so it can rebind things,
% including the definition of \endlisp (which normally is erroneous).

% Amount to narrow the margins by for \lisp.
\newskip\lispnarrowing \lispnarrowing=\parindent

% This is the definition that ^M gets inside \lisp
% phr: changed space to \null, to avoid overfull hbox problems.
{\obeyspaces%
\gdef\lisppar{\null\endgraf}}

% Cause \obeyspaces to make each Space cause a word-separation
% rather than the default which is that it acts punctuation.
% This is because space in tt font looks funny.
{\obeyspaces %
\gdef\sepspaces{\def {\ }}}

\newskip\aboveenvskipamount \aboveenvskipamount= 0pt
\def\aboveenvbreak{{\advance\aboveenvskipamount by \parskip
\endgraf \ifdim\lastskip<\aboveenvskipamount
\removelastskip \penalty-50 \vskip\aboveenvskipamount \fi}}

\def\afterenvbreak{\endgraf \ifdim\lastskip<\aboveenvskipamount
\removelastskip \penalty-50 \vskip\aboveenvskipamount \fi}

\newenvironment{example}{\aboveenvbreak
	\hfuzz=12truept % Don't be fussy
	\sepspaces % Make spaces be word-separators rather than space tokens.
	\singlespace % Single space lines
	% The following causes blank lines not to be ignored
	% by adding a space to the end of each line.
	\let\par=\lisppar
	\parskip=0pt \advance \rightskip by \lispnarrowing 
	\advance \leftskip by \lispnarrowing
	\parindent=0pt
	\let\exdent=\internalexdent
	\obeyspaces \obeylines \tt \rawbackslash \bgroup\samepage
	\def\next##1{}\next}{\egroup\afterenvbreak}

%same is the same as an enclosed \samepage
\newenvironment{same}{\bgroup\samepage}{\egroup}

% This is \begin{display}; same as \begin{lisp} except use roman font.

\newenvironment{display}{\aboveenvbreak
	% Make spaces be word-separators rather than space tokens.
	\sepspaces %
	% Single space lines
	\singlespace %
	% The following causes blank lines not to be ignored
	% by adding a space to the end of each line.
	\let\par=\lisppar
	\parskip=0pt \advance \rightskip by \lispnarrowing 
	\advance \leftskip by \lispnarrowing
	\parindent=0pt
	\let\exdent=\internalexdent
	\obeyspaces \obeylines
		\def\next##1{}\next}{\afterenvbreak}

% This is \format; same as \lisp except use roman font and don't narrow margins

\newenvironment{format}{\aboveenvbreak
	% Make spaces be word-separators rather than space tokens.
	\sepspaces %
	\singlespace %
	% The following causes blank lines not to be ignored
	% by adding a space to the end of each line.
	\let\par=\lisppar
	\parskip=0pt \parindent=0pt
	\obeyspaces \obeylines
	\def\next##1{}\next}{\afterenvbreak}

\message{cross reference,}
% Define cross-reference macros

% \xref and \pxref generate cross references to specified points.

\def\pxref #1{see \xrefX [#1,,,,,,,]}
\def\xref #1{See \xrefX [#1,,,,,,,]}

%\def\xrefX [#1,#2,#3,#4,#5,#6]{section \ref{#1} [%
%{\setbox0=\hbox{\losespace#3{}}%
%\ifdim \wd0 =0pt \losespace#1{}\else\unhbox0\fi}],
%page\tie \pageref{#1}}

\def\xrefX [#1,#2,#3,#4,#5,#6]{%
\setbox1=\hbox{\i{\losespace#5{}}}%
\setbox0=\hbox{\losespace#3{}}%
\ifdim \wd0 =0pt \setbox0=\hbox{\losespace#1{}}\fi%
\ifdim \wd1 >0pt%
section \unhbox0{} in \unhbox1%
\else%
\ref{#1} [\unhbox0], page\tie \pageref{#1}%
\fi }

\message{sectioning,}
% Define chapters, sections, etc.

\def\@chapter[#1]#2{\ifnum \c@secnumdepth >\m@ne
 \refstepcounter{chapter}
\ifx\lastnode\relax\else\label{\lastnode}\fi
\let\lastnode=\relax
 \typeout{\@chapapp\space\thechapter.}
 \addcontentsline{toc}{chapter}{\protect
 \numberline{\thechapter}#1}\else
 \addcontentsline{toc}{chapter}{#1}\fi
 \chaptermark{#1}
 \addtocontents{lof}{\protect\addvspace{10pt}}
\addtocontents{lot}{\protect\addvspace{10pt}} \if@twocolumn
\@topnewpage[\@makechapterhead{#2}] 
 \else \@makechapterhead{#2}
 \@afterheading \fi} 
\def\@schapter#1{\if@twocolumn \@topnewpage[\@makeschapterhead{#1}]
 \else \@makeschapterhead{#1} 
 \@afterheading\fi}

\def\@sect#1#2#3#4#5#6[#7]#8{\ifnum #2>\c@secnumdepth
     \def\@svsec{}\else 
     \refstepcounter{#1}
\ifx\lastnode\relax\else\label{\lastnode}\fi
\let\lastnode=\relax
	\edef\@svsec{\csname the#1\endcsname\hskip 1em }\fi
     \@tempskipa #5\relax
      \ifdim \@tempskipa>\z@ 
        \begingroup #6\relax
          \@hangfrom{\hskip #3\relax\@svsec}{\interlinepenalty \@M #8\par}
        \endgroup
       \csname #1mark\endcsname{#7}\addcontentsline
         {toc}{#1}{\ifnum #2>\c@secnumdepth \else
                      \protect\numberline{\csname the#1\endcsname}\fi
                    #7}\else
        \def\@svsechd{#6\hskip #3\@svsec #8\csname #1mark\endcsname
                      {#7}\addcontentsline
                           {toc}{#1}{\ifnum #2>\c@secnumdepth \else
                             \protect\numberline{\csname the#1\endcsname}\fi
                       #7}}\fi
     \@xsect{#5}}

\def\smallbook{
\global\lispnarrowing = 0.3in
\global\baselineskip 12pt
\global\parskip 3pt plus 1pt
\global\hsize = 5in
\global\doublecolumnhsize=2.4in \global\doublecolumnvsize=15.0in
\global\vsize=7.5in
\global\tolerance=700
\global\hfuzz=1pt

\global\pagewidth=\hsize
\global\pageheight=\vsize
\global\font\ninett=cmtt9

\global\let\smalllisp=\smalllispx
\global\let\smallexample=\smalllispx
\global\def\Esmallexample{\Esmalllisp}
}

%% For a final copy, take out the rectangles
%% that mark overfull boxes (in case you have decided
%% that the text looks ok even though it passes the margin).
\def\finalout{\overfullrule=0pt}

\message{and turning on texinfo input format.}

{\catcode`\@=0

% \rawbackslashxx output one backslash character in current font
{\catcode`\\=\other
@gdef@rawbackslashxx{\}}

% \rawbackslash redefines \ as input to do \rawbackslashxx.
{\catcode`\\=\active
@gdef@rawbackslash{@let\=@rawbackslashxx }}
}

% \back outputs one backslash in fixed width font.
\gdef\back{\tt\rawbackslashxx\ignorespaces}

\def\document{\endgroup
  \@colht\textheight  \@colroom\textheight \vsize\textheight
   \columnwidth\textwidth \@clubpenalty\clubpenalty
   \if@twocolumn \advance\columnwidth -\columnsep 
      \divide\columnwidth\tw@ \hsize\columnwidth \@firstcolumntrue 
   \fi
  \hsize\columnwidth \linewidth\hsize
  \begingroup\@floatplacement\@dblfloatplacement
   \makeatletter\let\@writefile\@gobbletwo
   \@input{\jobname.aux}\endgroup 
  \if@filesw \immediate\openout\@mainaux=\jobname.aux
    \immediate\write\@mainaux{\relax}\fi
  \def\do##1{\let ##1\@notprerr}
  \@preamblecmds
  \let\do\noexpand
\@normalsize\everypar{}\nospecials}


% Here is a list of the characters that have been specially catcoded:
\def\dosomespecials{\do\$\do\&\do\#\do\%\do\~\do\^}

{\catcode`\"=\active
\catcode`\>=\active
\catcode`\<=\active
\catcode`\_=\active
\gdef\nospecials{
\catcode`\"=\active
\gdef\activedoublequote{{\tt \char '042}}
\let"=\activedoublequote
\catcode`\>=\active
\gdef\activegtr{{\tt \char '076}}
\let>=\activegtr
\catcode`\<=\active
\gdef\activeless{{\tt \char '074}}
\let<=\activeless
\catcode`\_=\active
\gdef\activeunderscore{{\tt \char '137}}
\let_=\activeunderscore
\@noligs\let\do\@makeother \dosomespecials}
}
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib
LIBS = $(MDEP_LIBS)
LIBLINK = ../bin

default:	

all:	

realclean:	clean
	$(RM) -f Makefile *.bak

SHAR_EOF
if test -f 'p4.info'
then
	echo shar: over-writing existing file "'p4.info'"
fi
cat << \SHAR_EOF > 'p4.info'
Info file: p4.info,    -*-Text-*-
produced by latexinfo-format-buffer
from file: p4.tex




File: p4.info  Node: Top, Prev: (dir), Up: (dir), Next: Introduction


* Menu:

* Introduction::        
* Structure of the Distribution Directory::
* Installing p4::       
* Getting Started::     
* Specifying Processes in the Procgroup File::
* Developing a Simple p4 Program::
* Command-Line Arguments::
* The p4 Function Library::
* p4 Functions for Managing Processes and Clusters::
* Functions for Message Passing::
* Functions for Shared Memory::
* Functions for Timing p4 Programs::
* Functions for Debugging p4 Programs::
* Miscellaneous Functions::
* Fortran Interface::   
* Faster Startup with the Secure Server::
* Utilities for Managing a p4 Session::
* Creating Logfiles for Upshot::
* Machine-Specific Notes::
* Some Common Problems and their Solutions::
* Concept Index::       
* Function Index::      



File: p4.info  Node: Introduction, Prev: 	, Up: Top, Next: Structure of the Distribution Directory


Introduction
============


P4 is a library of macros and subroutines developed at Argonne National
Laboratory for programming a variety of parallel machines.  Its
predecessor was the m4-based "Argonne macros" system described in the
Holt, Rinehart, and Winston book {Portable Programs for Parallel
Processors}, by Lusk, Overbeek, et al., from which p4 takes its
name[lusk-overbeek:p4-book].  The current p4 system maintains the
same basic computational models described there (monitors for the
shared-memory model, message-passing for the distributed-memory model,
and support for combining the two models) while significantly increasing
ease and flexibility of use. *Note Getting Started:: for a simple example.

P4 is intended to be portable, simple to install and use, and efficient.  It
can be used to program networks of workstations, advanced parallel
supercomputers like the Intel Touchstone Delta and the Alliant Campus
HiPPI-based system, and single shared-memory multiprocessors.  It has
currently been installed on the following list of machines: Sequent Symmetry
(Dynix and PTX), Encore Multimax, Alliant FX/8, FX/800, and FX/2800, Cray
X/MP, Sun, NeXT, DEC, Silicon Graphics, HP, and IBM RS6000 workstations,
Stardent Titan, BBN GP-1000 and TC-2000, Intel IPSC/860, Intel Touchstone
Delta, Alliant Campus, and Thinking Machines' CM-5.  It will soon be ported to
to the Intel Paragon.  It is not difficult to port to new systems.  Although
p4 tries to be completely portable, there are a small number of
specific exceptions (*Note Machine-Specific Notes::) that may need to be taken
into account on a given machine.

You can obtain the complete distribution of p4 by anonymous ftp from
`info.mcs.anl.gov'.  Take the file `p4.tar.Z' from the
directory `pub/p4'.  The distribution contains all source code,
installation instructions, this reference manual, and a collection of
examples in both C and Fortran. `Alog' is included in the
distribution with p4.  The file `upshot.tar.Z' contains display
facilities that can be used with p4 and other systems.

To ask questions about p4, report bugs, contribute examples, etc., you can
send mail to `p4@mcs.anl.gov'.

The current release is version 1.2.  You can check which version of the source
code you have by looking at the file `lib/p4_patchlevel.h' in the
distribution.  You can check which version of the object code you have linked
to by running any p4 program with the command-line option `-p4version'
(*Note Command-Line Arguments::).

Salient features of the current release of p4 include: 
   * `xdr' support for heterogeneous networks
   * manual enhanced and converted to latex format
   * Emacs info version of the manual for on-line help
   * SYSV IPC support added for several machines (for shared-memory
       multiprocessing on workstations that support multiple processors)
   * instrumentation added for automatic logging/tracing
   * automatic or user control of message-passing/buffer-management
   * high-resolution clock support for several machines
   * error/interrupt handling
   * an optional secure server for quick startup


A useful companion system is the `alog/upshot' logging and X-based trace
examination facility.  (*Note Creating Logfiles for Upshot::.)



File: p4.info  Node: Structure of the Distribution Directory, Prev: Introduction, Up: Top, Next: Installing p4


Structure of the Distribution Directory
=======================================



The p4 source code distribution contains the following files and
subdirectories:


Makefile     
      The makefile for making the p4 system, doing the installation,
       and making makefiles for user applications.
OPTIONS     
      A file controlling various compile-time options, such as
       whether System V shared-memory operations are to be enabled, whether system
       debug message printing is to be enabled, and whether automatic
       instrumentation of p4 operations for the `upshot' logging and tracing
       program is to be done.
README     
      General instructions.
alog     
      Source code for the ALOG tracing package.
bin     
      Scripts for starting and killing servers, killing runaway p4
       processes, merging `upshot' logfiles, and other useful utilities.
contrib     
      Examples contributed by p4 users.
contrib_f     
      Fortran examples contributed by users.
doc     
      The man page, together with this manual and supporting files.
include     
      The include directory for making p4 applications.  Most of
       these are (hard) links into the `lib' directory.
lib     
      The source code for the p4 system.
lib_f     
      The Fortran interface for p4.  
messages     
      A basic set of message-passing examples in C.
messages_f     
      A basic set of message-passing examples in Fortran.
monitors     
      A basic set of shared-memory examples in C.
servers     
      The secure and insecure servers.
usc     
      The portable microsecond clock routines.
util     
      Assorted supporting files, particularly for making the p4
       distribution. 



File: p4.info  Node: Installing p4, Prev: Structure of the Distribution Directory, Up: Top, Next: Getting Started


Installing p4
=============



In this section we describe how to install the p4 library, either for
your own personal use or for the use of everyone at your site.  In the
first case you do not need any super-user privileges.  In the second
case, you may or may not, depending on how things are configured at
your site.  We also describe how to install and run the examples that
come with p4, the online help system (this manual as an emacs info-file)
and how to build a working directory for your own programs yet share
the installed copy of p4 with other users.


* Menu:

* Installing the p4 System::
* Installing the Documentation::
* Examples included with the Distribution::



File: p4.info  Node: Installing the p4 System, Prev: 	, Up: Installing p4, Next: Installing the Documentation


Installing the p4 System
------------------------


To build p4, position yourself in the `p4' directory and type:
     make all MACHINE=<machine>

where <machine> is one of the machine names listed in
`p4/util/machines', currently:


SUN     
      Sun-3, Sun386i, Sparc-1, or Sparc-2 workstations
     
HP     
      HP workstations
     
DEC5000     
      Dec 5000 workstations
     
NEXT     
      68030- or 68040-based NeXT workstations
     
RS6000     
      IBM RS 6000 series workstations
     
IBM3090     
      IBM 3090 running IBM's version of UNIX, AIX
     
BALANCE     
      Sequent Symmetry shared-memory multiprocessor
     
SYMMETRY     
      Sequent Symmetry shared-memory multiprocessor
     
SYMMETRY_PTX     
      Sequent Symmetry shared-memory multiprocessor PTX OS
     
MULTIMAX     
      Encore Multimax shared-memory multiprocessor
     
GP_1000     
      BBN GP-1000
     
TC_2000     
      BBN TC-2000
     
TC_2000_TCMP     
      BBN TC-2000 with the TCMP message-passing library
     
IPSC860     
      Intel IPSC/860 (nodes only)
     
IPSC860_SOCKETS     
      Intel IPSC/860 with socket libraries on the nodes
     
DELTA     
      Intel DELTA 
     
TITAN     
      Stardent Titan
     
SGI     
      Silicon Graphics workstations
     
CRAY     
      Cray X/MP
     
FX8     
      Alliant FX/8
     
FX2800     
      Alliant FX/2800 or FX/800
     
FX2800_SWITCH     
      Alliant FX/2800 or FX/800, with CAMPUS HiPPI switch
     
KSR     
      Kendall Square KSR-1
     
CM-5     
      Thinking Machines' CM-5






For example:
     make all MACHINE=SYMMETRY

The `all' is optional, for example
     make MACHINE=SEQUENT

This will create a machine-dependent `Makefile' in each subdirectory,
make the p4 library, and compile and link a subset of the examples.

To add a new machine type, or to change the characteristic parameters
associated with an existing one, you can edit the file 
`p4/util/defs.all'.

To save disk space, various intermediate object files can be removed with
     make clean

The system can be restored to its original, machine-independent state with
     make realclean

Note that this removes the machine-dependent Makefiles in each directory, so
the operation is not idempotent.

It is also possible to install (or clean) only some of the directories:
     make all MACHINE=SUN DIRS=messages
     make clean DIRS='monitors messages'

To install only the Makefiles in all subdirectories, use:
     make makefiles MACHINE=<machine>

directory everything that is needed to compile and link p4 programs, 
do:

     make install INSTALLDIR=<dir>

minimal set of directories, copy the relevant `.a' and `.h' files
into it, and test the installation by mking a small set of examples.


*Note Getting Started:: for instructions on how to run some example
programs after you have installed p4.

File: p4.info  Node: Installing the Documentation, Prev: Installing the p4 System, Up: Installing p4, Next: Examples included with the Distribution


Installing the Documentation
----------------------------






The directory `p4/doc' contains this manual as well as files that
require installation.  This manual was prepared with the
`latexinfo' package from GNU emacs; thus it can be made available online
through `info'.  The files in `p4/doc' are:

p4.tex     
      the latex source for this manual, which uses the latexinfo style
     
latexinfo.sty     
      the sytle file needed to latex this manual
     
p4.info     
      the `info' version of this manual, ready to be put
     into the directory where `info' files are kept at your site.  Check
     the value of your `emacs' variable INFO-DIRECTORY.
     
p4.txt     
      plain ascii text of the manual, in case nothing else works.
     
p4.1     
      unix man page for the p4 library
     
p4f.1     
      unix man page for the Fortran interface to p4
     


The Postscript version of this manual is available by anonymous `ftp'
from `info.mcs.anl.gov', in the directory `pub/p4'.  The file to
get (in binary mode) is `p4man.ps.Z'.


File: p4.info  Node: Examples included with the Distribution, Prev: Installing the Documentation, Up: Installing p4, Next: 	


Examples included with the Distribution
---------------------------------------



A good way to see how various p4 functions are used is to look at the example
programs included in the distribution.  The `p4/monitors' directory
contains shared-memory examples written in C that use monitors, including one
instrumented with ALOG.  The `p4/messages' subdirectory contains
message-passing examples written in C.  The programs in `p4/messages_f'
are Fortran message-passing examples, and the `p4/contrib' and
`p4/contrib_f' directories contain a number of miscellaneous examples
contributed by users.  In each directory there is a `README' that
describes the individual examples.


File: p4.info  Node: Getting Started, Prev: Installing p4, Up: Top, Next: Specifying Processes in the Procgroup File


Getting Started
===============



The easiest way to get started with p4 is to play with some of the
sample programs provided with the system.

* Menu:

* A Message-Passing Example::
* Analysis of the Program::



File: p4.info  Node: A Message-Passing Example, Prev: 	, Up: Getting Started, Next: Analysis of the Program


A Message-Passing Example
-------------------------


We will begin with a message-passing example in the sub-directory named
`p4/messages'.  The code for the program is in the files `sr_test.c'
and `sr_user.h'.


 
Program Description
-------------------


As the name implies, this program is an example of p4's send/receive
functionality.  Briefly, it is a simple program that runs a master
process and some slave processes.  The master and the set of slaves
form a ring of processes in which the master reads a message from stdin
and sends a copy of the message to the first slave, which passes it on;
the last slave passes the message back to the master.  If the master
receives an undamaged copy of the message, it assumes that all went
well, and reads another message.  Note that the ring of processes is a
logical structure in which each process assumes that its
predecessor in the ring is the process with the next lower id, and
its successor is the process with the next higher id.  The master
has id 0 (zero) and has the process with the largest id as its
predecessor.



File: p4.info  Node: Analysis of the Program, Prev: A Message-Passing Example, Up: Getting Started, Next: 	


Analysis of the Program
-----------------------


The first executable p4 statement in a program should be:
     p4_initenv(&argc,argv); 

This initializes the p4 system and allows p4 to extract any command
line arguments passed to it, e.g. debugging parameters.  

Similarly, the last executable p4 statement in a program should be:
     p4_wait_for_end(); 

This waits for termination of p4 processes and performs some cleanup
operations.

The procedure `p4_get_my_id' returns the unique integer id assigned
to the calling process by p4.

The statement:
     p4_create_procgroup();

reads a procgroup file that the user builds and creates the set of
slaves described in that file.  Obviously this statement must be
executed before any slaves can be assumed to exist.  This procedure
is the method you must use to create processes that do message-passing.

The procedure `p4_clock' returns an integer that represents
wall-clock time in milliseconds.  It is typically used to retrieve the
time before and after some work, the difference representing the time to
do that work.  Note that there is also a `p4_ustimer' that is useful on
those machines that support a microsecond timer.

The procedures `p4_send' and `p4_sendr' are two of several 
p4 procedures that are available for sending messages to other processes.
They take as arguments the message type, the id of the "to" process,
the address of the message, and the message length.

The procedure `p4_recv' receives a message from another process and
sets the values of all four parameters.  `P4_recv' will automatically
retrieve a buffer in which to place a received message, thus
`p4_msg_free' may be called to free that buffer when it is no longer
needed.

The procedure `p4_num_total_slaves' is one of several procedures that
the user can invoke to determine information about the current execution.

To run this program, you need to create a procgroup file that describes
where all slave processes are to be executed 
(*Note Specifying Processes in the Procgroup File::).  We will assume that
you have an example procgroup file (named `procgroup') in the 
`p4/messages' directory, and can run `sr_test' by merely typing:
     sr_test

If the procgroup file is elsewhere, then you must type:
     sr_test -pg  {pathname_of_procgroup_file}


Another example that is made by default is the program `systest'.  It
tests a number of the message-passing features of p4.


File: p4.info  Node: Specifying Processes in the Procgroup File, Prev: Getting Started, Up: Top, Next: Developing a Simple p4 Program


Specifying Processes in the Procgroup File
==========================================



The procgroup file is the only portion of the interface that is very
likely to change through multiple versions of p4.  As new architectures
are supported, it is hoped that we can merely alter the procgroup file
format to reflect any new features.  (Of course new procedure calls may 
also be required, but existing procedure calls will remain unchanged 
when possible).

The current format of a procgroup file is as follows:
     local n [full_path_name] [loginname]
     remote_machine n full_path_name [loginname]
     .
     .
     .


On cube and mesh architectures, the program is started via some
special command executed from the host machine.  In such cases, the
procgroup file name can be specified to the special command line along
with the program name (see for example the `runcube' and `rundelta'
shell scripts in the `p4/messages' subdirectory).  In those cases 
where no special command is required, no special handling is required 
for the procgroup filename.

The first line of a procgroup file must be "local n" where n is the
number of slave processes that are in the same cluster as the master.
The full path name on the "local" line is ignored on machines other
than cube and mesh machines.  The subsequent lines contain either
three or four fields:
  1. the name of a remote machine on which slave processes are to be created.
  2. the number of slaves that are to be created on that machine, 
     i.e. be in the same cluster (note that on machines that support it, 
     the processes in a cluster will share memory)
  3. the full path name of the executable slave program
  4. optionally, the user login name on the remote machine, if different from 
     that on the host machine.


As an example, let's assume that you have a network of three Sun 
workstations named sun1, sun2, and sun3.  We will also assume that you 
are working on sun1 and plan to run a master process there.  
If you would like to run one process on each of the other Suns, then you 
might code a procgroup file that looks like:

         # start one slave on each of sun2 and sun3
         local 0 
         sun2  1  /home/mylogin/p4pgms/sr_test
         sun3  1  /home/mylogin/p4pgms/sr_test


Lines beginning with `#' are comments.

Next, let's assume that you have a Sequent Symmetry (named symm) and an
Encore Multimax (named mmax).  We will also assume that you are working
on symm, and plan to run the master there.  If you would like to run
two processes on symm (in addition to the master) and two on mmax, then 
you might code a procgroup file that looks like:

         local 2 
         mmax  2  /mmaxfs/mylogin/p4pgms/sr_test


P4 also permits you to treat the symmetry as a remote machine even when 
you are running the master there.  Thus, you might code a procgroup file 
as follows:

         local 2 
         symm  2  /symmfs/mylogin/p4pgms/sr_test
         mmax  2  /mmaxfs/mylogin/p4pgms/sr_test


In this example, there are seven processes running.  Five of the
processes are on symm, including the master.  Two of the processes on
symm are in the master's procgroup and two are running in a separate
procgroup as if they were on a separate machine.  Of course, the last
two are running on mmax.

Some notes about the contents of the procgroup file should be made at
this point.  First, the value of `n' on the local line can be zero,
i.e. the master may have no local slaves.  Second, the local machine
may be treated as if it is a remote machine by merely entering it in
some line as a remote machine.  Third, a single machine may be treated
as multiple remote machines by having the same remote machine name
entered on multiple lines in the procgroup file.  Fourth, if a single
machine is listed multiple times, those processes specified on each
line form a single cluster (share memory).  Fifth, the cluster size
specified for a uniprocessor should be 1, because all slaves in a
cluster are assumed to run in parallel and to share memory.

We refer to the original (master) process as the "big master".  The
first process created in each cluster is the "remote master" or the
"cluster master" for that cluster.  
All p4-managed processes (see the procedure `p4_create_procgroup') 
have unique integer id's beginning with 0.
The processes within a cluster are numbered consecutively.



File: p4.info  Node: Developing a Simple p4 Program, Prev: Specifying Processes in the Procgroup File, Up: Top, Next: Command-Line Arguments


Developing a Simple p4 Program
==============================


The real fun associated with any computing environment arrives when you
actually type in a program and run it yourself.  We will assume that
you have successfully installed p4 on your own system and are ready to
write a small program, compile it, and run it.

* Menu:

* A Minimal Example::   
* A More Complicated Example::



File: p4.info  Node: A Minimal Example, Prev: 	, Up: Developing a Simple p4 Program, Next: A More Complicated Example


A Minimal Example
-----------------


We will start with a tiny program in which the slave processes do no work, and
then expand its capabilities.  Edit a file called `p4simple.c' and type:

     
         #include "p4.h"
     
         main(argc,argv)
         int argc;
         char **argv;
         {
             p4_initenv(&argc,argv);
             if (p4_get_my_id() == 0)
                 p4_create_procgroup();
             slave();
             p4_wait_for_end();
         }
     
         slave()
         {
             printf("Hello from %d++",p4_get_my_id());
         }
     


This is one of the simplest p4 programs that you can write.  Let's
examine it.  The `#include "p4.h"' statement must appear in all
programs that use any p4 features.  The procedure `p4_initenv'
must be invoked before any other p4 procedures, and
`p4_wait_for_end' must be invoked after all p4 processing is
completed.  The `p4_get_my_id' returns a unique integer id for
each process, beginning with 0.  The procedure
`p4_create_procgroup' should only be invoked once (by process 0),
and is responsible for creating all other processes.  The way in which
`p4_create_procgroup' determines how many other processes there
should be, and where they should run, will be discussed shortly.

All processes that this program executes invoke the slave procedure,
including process 0.  Thus, in this program, the master process acts
just like all other processes once it gets the environment
established.

To understand how things get started, let's consider two separate
situations.  In the first situation, all processes are running on a
single machine.  Then, when process 0 starts, it executes the
`p4_create_procgroup' procedure to start all other slaves.  The other
slaves are started on the same machine by means of a UNIX {fork},
and they immediately invoke a procedure named slave.  Thus, these local
slaves do not ever execute the main procdure.

In the second situation, there may be slaves running both on the same
machine as process 0, and slaves running on other machines as well.
In this situation, the first slave running on a remote machine will
need to execute the main procedure.  It will discover that it is not
process 0.  However, as part of initialization, process 0 will
direct it to fork any additional slaves required on the same machine.


In some ways, the above example can be used a a prototype for all p4 programs,
just by varying the content of the `slave' routine.  The routine name
`slave' is built into p4 in order to enable the same code to be run on
shared-memory machines via `fork' or on remote machines via `rsh'.
Different wrappers are automatically put around the `slave' code.  These
wrappers depend on the user code being called `slave'.  Thus your p4
program must contain a subroutine called `slave' in order to link
properly. 


File: p4.info  Node: A More Complicated Example, Prev: A Minimal Example, Up: Developing a Simple p4 Program, Next: 	


A More Complicated Example
--------------------------


Now, let's make the slave process a little bit more interesting.  Let's
assume that we have `nprocs' slaves with ids 
0, 1, 2, ... `nprocs' -1.  And, we want to write a program in which
every process sends a single message to every other slave, and then
receives a message from every other slave.  We might alter the code for
the slave procedure to be the following:

         slave()
         {
             char *incoming, *msg = "hello";
             int myid, size, nprocs, from, i, type;
     
             myid = p4_get_my_id();
             nprocs = p4_num_total_ids();
             for (i=0; i < nprocs; i++)
             {
                 if (i != myid)
                     p4_send(100, i, msg, strlen(msg)+1);
             }
             for (i=0; i < nprocs - 1; i++)
             {
                 type = -1;
                 from = -1;
                 incoming = NULL;
                 p4_recv(&type,&from,&incoming,&size);
                 printf("%d received msg=:%s: from %d++",myid,incoming,from);
                 p4_msg_free(incoming);
             }
         }


This program demonstrates several features of p4's support for
message-passing.  Before we get into the specifics however, let's
examine the overall logic of the program.  Each process determines its
own id and the total number of processes executing in this run
(including process 0).  Then, in the first for-loop, each process sends
a single message to each of the other processes.  Finally, in the
second for-loop, each process receives a message from each of the other
processes.

The `p4_send' call requires 4 arguments:
   * a message type (arbitrarily chosen to be 100 here)
   * the id of the process to receive the message
   * the message itself
   * the size of the message


The use of `p4_recv' is slightly more complicated.  First, we
assign -1 to each of the parameters type and from.  This is done
because -1 represents a wildcard value indicating we are willing to
receive a message of any type from any process.  Here, we could have
coded type to be 100, and specified from equal to the value of i each
time through the loop (skipping our own id).  By setting incoming to
NULL, we have also indicated to `p4_recv' that we do not have a
buffer in which to place the received message, so `p4_recv' should
obtain a buffer for us and place the message in that buffer.
`p4_recv' treats these three parameters as both input and output
values.  Thus, it alters the value of each such that type and from
indicate the type of message received and the id of the process that
sent it.  The value of `incoming' is altered to point to the
buffer where the message was placed.  The `size parameter' is
strictly an output parameter and iindicates the size of the received
message.  It is possible for the user to provide his own buffer; this
will be demonstrated later.


Finally, note that `p4_msg_free' frees the message buffer obtained by
`p4_recv.' The procedure `p4_msg_free' should be called only after
the contents of the message are no longer needed.  `P4_msg_free' should
be used to free these buffers because, although a user only sees the data
portion of a message, p4 internally represents a message as a structured data
item.

To compile and link this program for execution, you need to create a
makefile.  We will assume that you have installed p4 in
`/usr/local/p4' and that you have typed the program above into a
file name `p4simple.c' in the directory
`/home/mylogin/p4pgms'.

To build your makefile, copy the file
         /usr/local/p4/messages/makefile.proto

into your working directory.  This is a prototype makefile that
contains machine-independent information, and which p4 can use to build a
machine-specific makefile for your program.  This prototype makefile contains
information about several sample programs that demonstrate
message-passing in p4.  If you edit this file, you will see information
for making a program named `sr_test'.  Do a global change of
`sr_test' to `p4simple'.  You should also change the value of
`P4_HOME_DIR'.  It should contain the full
pathname of the p4 system, e.g. `/usr/local/p4'.  Now change
directories to `/usr/local/p4' and type:

     make makefiles MACHINE=<machine_type> DIRS=/home/mylogin/p4pgms

where `<your_machine_type>' is the machine type that you specified when
you installed p4 on your machine.  Now, you should be able to change
back to your directory and see a file named Makefile there.  
You should then be able to type:

         make p4simple


There is one last piece missing before you can execute your program.
Recall that `p4_create_procgroup' needs to know how many processes
to start and where to start them; it reads a file (called a
"procgroup file") to gather this information.  p4 always assumes
that you have a master process, and that you describe the slave
processes (process groups) in the procgroup file.  You can name a
procgroup file any name you choose, but `procgroup' is the default
name.  The information contained in procgroup files can get fairly
involved, but if you have a computer that supports shared memory among
processes, then you can code a very simple example at first.

Let us suppose first that you want to run your program on a network of
workstations.  Then your procgroup should look something like:

         local O
         some.network.machine 1 /home/me/p4progs/p4simple


This file indicates that you wish to run only the master on the local machine
(the one you are logged into when you execute the program) and one slave on
the machine `some.network.machine'.

Now, all you have to do to run your program is type:

         p4simple


You should see a line printed each time a process receives a message
from another process (on some machines, there may be a restriction that
only one process can do I/O, however such restrictions are not
common).  Experiment by changing the number of slaves indicated in the
procgroup file.

You may notice that even a small p4 program becomes large when linked
with the p4 library.  You might consider using `strip' to reduce
the size or removing `-g' from the CFLAGS in the makefile.


File: p4.info  Node: Command-Line Arguments, Prev: Developing a Simple p4 Program, Up: Top, Next: The p4 Function Library


Command-Line Arguments
======================



The command-line arguments to a p4 program are all optional.
     -help           get this information and the version number, then exit
     -pg             procgroup_file
     -dbg            set debug level 
     -rdbg           set remote debug level
     -gm             set globmemsize
     -dmn            provide local domainname
     -outfile        output file for master
     -routfile       output file prefix for remote masters
     -ssport         port# private port number for secure server
     -p4log          enable internal p4 logging by alog
     -p4version      print the current p4 version number




File: p4.info  Node: The p4 Function Library, Prev: Command-Line Arguments, Up: Top, Next: p4 Functions for Managing Processes and Clusters


The p4 Function Library
=======================


* Menu:

* Overview of the Library::
* Return Codes from p4 Functions::



File: p4.info  Node: Overview of the Library, Prev: 	, Up: The p4 Function Library, Next: Return Codes from p4 Functions


Overview of the Library
-----------------------


In the following sections, we provide details for each p4 function in the
library.  The procedures are gathered into the following groups:

   * Functions for managing processes and clusters
   * Functions for message passing
   * Functions for shared memory
   * Functions for timing p4 programs
   * Functions for debugging p4 programs
   * Miscellaneous functions
   * Fortran interface functions



File: p4.info  Node: Return Codes from p4 Functions, Prev: Overview of the Library, Up: The p4 Function Library, Next: 	


Return Codes from p4 Functions
------------------------------



Most p4 functions return -1 if an error occurs.  Some, however, call the
function `p4_error' when severe errors occur.  This function prints a
message and then attempts to terminate all of the user's processes
*Note Functions for Debugging p4 Programs::.


File: p4.info  Node: p4 Functions for Managing Processes and Clusters, Prev: The p4 Function Library, Up: Top, Next: Functions for Message Passing


p4 Functions for Managing Processes and Clusters
================================================


In some situations a p4 procedure will give an error message and then
exit.  This is typically done as a result of a failed system call and
handled by calling the p4 procedure named `p4_error' that examines the
return values from socket procedures, etc.  Most of the time however,
the procedures simply return a value.  Some of the procedures return no
value and thus are declared to return `VOID'.  Some of the
procedures return either a pointer to a character string or `NULL';
`NULL' indicates an error.  The remaining procedures return an
integer value; (-1) indicates an error.


* Menu:

* Functions for Process Management::
* Functions for Cluster Management::



File: p4.info  Node: Functions for Process Management, Prev: 	, Up: p4 Functions for Managing Processes and Clusters, Next: Functions for Cluster Management


Functions for Process Management
--------------------------------



In this section we describe the p4 functions needed for basic creation and
termination of processes.


     int p4_initenv(argc,argv)
     int *argc;
     char **argv;

should be called by your program before an attempt is made to use
any p4 procedures or data areas.  We suggest making it the first executable
statement in your program.  `p4_initenv' parses the command line
arguments and extracts the ones intended for p4 ignoring all others (see the
discussion of command line arguments).  Note that you pass the address of
`argc' to `p4_initenv' so that it can actually remove its own
arguments before your program looks at them.


     int p4_create(fxn)
     int (*fxn)();
     
     
     int p4_create_procgroup()

There are two procedures that you can use to create processes in p4,
`p4_create_procgroup' and `p4_create'.  Processes created via
`p4_create' are said to be "user-managed" whereas those created
by `p4_create_procgroup' are "p4-managed".  The p4-managed
processes are automatically assigned unique id's (beginning with 0 for
the big master), they have message queues allocated for them so that
they can do message-passing, and they are able to run either on a
shared-memory multiprocessor with the creating process or they can run
on a separate machine.  Processes created via `p4_create' do not
have any of these advantages.  They must develop their own id's, they
cannot do message-passing, and they can only run on a shared-memory
multiprocessor with the creating process.  The only disadvantage of
`p4_create_procgroup' is that you must build a `procgroup'
file describing the set of required slave processes before the master
program begins execution.  This eliminates the possibility of
determining late in the execution exactly how many processes you want
to use to solve a problem.  Generally, this is not a problem,
especially since we can combine `p4_create_procgroup' and
`p4_create' in the following way: You can use
`p4_create_procgroup' to develop a network of processes that talk
to each other via messages.  Each of those processes can create further
processes to help it out as necessary.  The original set of processes
communicate with their local slaves through shared data areas and with
each other via message-passing.

`p4_create' receives one argument that is a pointer to a function.  It
creates a single new process that executes the indicated function.  The
new process may share data areas (in shared memory) with the parent
process.  However, the new process is not managed by the p4 system in
the sense that it is not assigned an id, it cannot pass messages, etc.
The only p4 procedure that deals with user-managed slaves is `p4_create'.
No other procedures are even aware of their existence.

`p4_create_procgroup' reads your `procgroup' file to determine the 
number of slave processes to create and where they are to be placed.  It
builds a procgroup table that describes all created processes and gives
a copy of the table to each process.  The processes then use the table
to discover how to communicate with each other (processes in a cluster
can send messages directly through shared memory or some other
vendor-specific mechanism), others communicate via sockets). 
An alternative method is to build the table in memory yourself and use
`p4_startup'.

The effect of `p4_create_procgroup' can be obtained in another way if a
system would prefer to use its own way of specifying the locations of
processes.  A user may allocate the procgroup data structure and then fill it
in "by hand" rather than by reading a file in p4 procgroup format.  The
following procedures support this method of starting processes.


     struct p4_procgroup *p4_alloc_procgroup()

allocates a procgroup data structure of the form described in `p4.h'.
The formats of individual entries (`p4_procgroup_entry') are given there
as well. 


     int 4_startup(pg)
     struct p4_procgroup *pg;

starts processes as specified by an an already-created procgroup data
structure allocated by `p4_alloc_procgroup' and filled in by the user
using the structures `p4_procgroup_entry' and `p4_procgroup'.



     VOID p4_wait_for_end()

is the p4 termination/cleanup procedure that you should invoke at the
end of every execution of a program that uses p4.  
It does some termination processing and then waits for slave processes 
to end.


     int p4_get_my_id()

returns an integer value representing the id of the process assigned by
the p4 system.  If the process is not a p4-managed process, the value
(-1) is returned.


     int p4_num_total_ids()

returns an integer value indicating the total number of ids started
by p4 in all clusters, including the big master and all remote masters.


     int p4_num_total_slaves()

returns an integer value indicating the total number of processes started
by p4 in all clusters, including all remote masters but not the big master.


File: p4.info  Node: Functions for Cluster Management, Prev: Functions for Process Management, Up: p4 Functions for Managing Processes and Clusters, Next: 	


Functions for Cluster Management
--------------------------------



The p4 system supports the "cluster" model of parallel computation, in
which subsets of processes share memory with one another, with the clusters
communicating via messages.  A procgroup file for a program written for the
cluster model might look like this:

         local 4
         alliant1.abc.edu     5 /home/me/myprog
         alliant2.abc.edu     5 /home/me/myprog
         encore.somewhere.edu 5 /usrs/me/myprog


This would specify a total of 20 processes, 5 (including the master) running
on the local machine (here assumed to be capable of supporting five processes
that share memory) together with 5 slaves each on three other shared-memory
machines.


     VOID p4_get_cluster_ids(start,end)     
     int *start;
     int *end;

receives pointers to two integers.  It places the p4-assigned id's of
the first and last ids within the current cluster into the two
arguments (including the remote master).


     int p4_get_my_cluster_id()

returns a unique id (relative to 0) within a cluster of p4-managed
processes.  Thus, a cluster master will always have a cluster id of 0.
It is not clear that a separate cluster id is really useful, but the
functionality is provided just in case.


     BOOL p4_am_i_cluster_master()

returns a BOOL value indicating whether the invoking process is the
"cluster master" process within its cluster.


     int p4_num_cluster_ids()

returns an integer value indicating the number of ids in the current
cluster as started by `p4_create_procgroup'.  



File: p4.info  Node: Functions for Message Passing, Prev: p4 Functions for Managing Processes and Clusters, Up: Top, Next: Functions for Shared Memory


Functions for Message Passing
=============================



P4 supports a set of send/receive procedures.  These procedures are
"generic" in the sense that they do not know whether a message must
travel across a network or through shared memory, or via some other
mechanism.  They depend on a lower-level set of procedures that handle
local or network (remote) communications.  By default, the messages
are assumed to be typed.  If the user wishes to use untyped messages,
he can hide the typing by coding some very simple C macros that always
use a single message type.


* Menu:

* Explicit Sending and Receiving of Messages::
* Global Operations::   



File: p4.info  Node: Explicit Sending and Receiving of Messages, Prev: 	, Up: Functions for Message Passing, Next: Global Operations


Explicit Sending and Receiving of Messages
------------------------------------------











     p4_send(type,to,msg,len)
     p4_sendr(type,to,msg,len)
     p4_sendx(type,to,msg,len,datatype)
     p4_sendrx(type,to,msg,len,datatype)
     p4_sendb(type,to,msg,len)
     p4_sendbr(type,to,msg,len)
     p4_sendbx(type,to,msg,len,datatype)
     p4_sendbrx(type,to,msg,len,datatype)
     
     int type, to, len, datatype;
     char *msg;


Each of these procedures sends a message.  The `type' argument is
an integer value chosen by the user to represent a message type.  The
`to' argument is an integer value that specifies the p4-id of the
process that should receive the message.  The `len' argument
contains the length of the message to be passed.  Note that some of the
procedures have a "b" in their name, e.g.  `p4_sendb'.  These
procedures assume that the msg is in a buffer that the user obtained
earlier via a `p4_msg_alloc'; otherwise, the buffer is assumed to
be in the user's local space, and may cause the message to be copied
internally.  The procedures with an "r" in the name do not return
until an acknowledgement is received from the `to' process (the
"r" stands for rendezvous).  Those procedures with an "x" in the
name take an extra argument (datatype) that specifies the type of data
in the message; these procedures will use that information to call XDR
for data conversion if the message is being passed to a machine of a
different architecture, i.e. where the internal representation may be
different.



     BOOL p4_messages_available(req_type,req_from)
     int *req_type,*req_from;

returns a BOOL value indicating whether the process has any messages available
or not.  The parameters `req_type' and `req_from' are both pointers
to integers; they are used as {both} input and arguments.  On input,
`req_type' has a value that indicates the type of message that the user
wishes to check for availability (-1 indicates any type).  The variable
`req_from' is used similarly to indicate who a message is desired from.



     int p4_recv(req_type,req_from,msg,len_rcvd)
     int *req_type,*req_from,*len_rcvd;
     char **msg;

pointer to a `char'.  If this value is NULL, then p4 will allocate the
buffer for the message according to its length.  That is, one need not know
ahead of time the length of a message being received.  If this value is not
NULL, then it points to a p4 message buffer that the user has obtained via
`p4_msg_alloc'.  The `len_rcvd' argument is a pointer to an integer
that is assigned the length of the received message.  `Req_type' and
`req_from' are both pointers to integers; they are used as both input and
arguments.  On input, `req_type' has a value that indicates the type of
message that the user wishes to receive (-1 indicates any type).  It will
block until a message of that type is available.  `Req_from' is used
similarly to indicate who a message is desired from.  One important note about
this procedure is that it obtains the area in which to place a message, and
the user must explicitly free that area when finished with it (see
`p4_msg_free').  There is an option available with `p4_recv' in
which the user can provide his own buffer rather than having p4 allocate it.
To do this, the user points `msg' to a buffer that he must obtain via a
call to `p4_msg_alloc' (see below).  Then he assigns `len_rcvd' a
value which is the length of that buffer.  In this case, `len_rcvd' is
both an input and output variable.  In addition, no `p4_msg_free' need be
performed if the same buffer is going to be re-used multiple times.



     char *p4_msg_alloc(len)
     int len;

obtains a pointer to a buffer area that can be used to receive a 
message.  This procedure should be used for this task because a 
message has hidden information which the user is unaware of and 
therefore should not use malloc to obatin the area.



     VOID p4_msg_free(m)
     char *m;

frees the message pointed to by `m.'  This procedure should be used
for this task because a message has hidden information which the
user is unaware of and therefore cannot be freed by the user.



File: p4.info  Node: Global Operations, Prev: Explicit Sending and Receiving of Messages, Up: Functions for Message Passing, Next: 	


Global Operations
-----------------



P4 supports a number of operations for dealing with all processes at once.


     p4_broadcastx(type, data, data_len, data_type)
     int type;
     char *data;
     int data_len, data_type;



     p4_broadcast(type, data, data_len)
     int type;
     char *data;
     int data_len;

provide the ability to broadcast messages like `p4_send' and
`p4_sendx'.


     p4_global_op(type,x,nelem,size,op,data_type) 
     int type;
     char *x;
     int size, nelem;
     int (*op)();
     int data_type;

where `op' is one of:
     p4_int_absmax_op()
     p4_int_absmin_op()
     p4_int_max_op()
     p4_int_min_op()
     p4_int_mult_op()
     p4_int_sum_op()
     p4_dbl_absmax_op()
     p4_dbl_absmin_op()
     p4_dbl_max_op()
     p4_dbl_min_op()
     p4_dbl_mult_op()
     p4_dbl_sum_op()
     p4_flt_absmax_op()
     p4_flt_absmin_op()
     p4_flt_max_op()
     p4_flt_min_op()
     p4_flt_mult_op()
     p4_flt_sum_op()


and `data_type' is one of `P4INT', `P4LNG', `P4FLT', or
`P4DBL'.

This collection of routines provide the ability to do a variety of
global operations.  See the example programs in subdirectory
`p4/contrib'.  They apply the commutative operation `op' globally 
to `x' on an element-by-element basis and broadcast the result to 
all nodes.  That is each process ends up with

        for (i=0; i<n; i++)
              x[i] = x[node 0][i] op x[node 1][i] op x[node 2][i] op ...

`op' should be of the form

          VOID op(char *x, char *y, int nelem)
          {
              data_type *a = (data_type *) x;
              data_type *b = (data_type *) y;
     
              while (nelem--)
                  *a++ operation= *b++;
          }

where `data_type' and `operation' are chosen appropriately.

The order in which nodes apply the operation is undefined (hence
`op' must be commutative). The communication may be internally
sub-blocked so the function `op' should not be hardwired to specific
vector lengths.

This is still a relatively primitive version, which gathers the necessary data
up a balanced binary tree and then uses `p4_broadcast' to send the
results back. 



     VOID p4_global_barrier(type)
     int type;

This procedure takes one argument which is the message type to be 
used for internal message-passing.  It causes the invoking process to
hang until all processes specified in the procgroup file have invoked 
the procedure.



File: p4.info  Node: Functions for Shared Memory, Prev: Functions for Message Passing, Up: Top, Next: Functions for Timing p4 Programs


Functions for Shared Memory
===========================




* Menu:

* Managing Shared and Local Memory::
* Shared Memory Data Types::
* Monitor-Building Primitives::
* Some Useful Monitors::



File: p4.info  Node: Managing Shared and Local Memory, Prev: 	, Up: Functions for Shared Memory, Next: Shared Memory Data Types


Managing Shared and Local Memory
--------------------------------



     char *p4_malloc(n)
     int n;

typically acts like the standard `malloc', but may be rewritten for user 
systems that require different operation.


     VOID p4_free(p)
     char *p;

typically acts like the standard `free', but may be rewritten for
user systems that require different operation.


     char *p4_shmalloc()

acts like the standard `malloc' except will obtain shared memory on
machines that support sharing memory among processes.  Compare with
`p4_malloc'.


     VOID p4_shfree()

frees memory obtained with `p4_shmalloc'.  Compare with `p4_free'.



File: p4.info  Node: Shared Memory Data Types, Prev: Managing Shared and Local Memory, Up: Functions for Shared Memory, Next: Monitor-Building Primitives


Shared Memory Data Types
------------------------





The abstraction provided by p4 for managing data in shared memory is
"monitors".  Good places to learn about the monitor concept in general are
[pbh:architecture] and [hoare:monitors].  The specific approach
taken by p4 is described in [lusk-overbeek:p4-book].  P4 provides several
useful monitors (`p4_barrier_t', `p4_getsub_monitor_t',
`p4_askfor_monitor_t') as well as a general monitor type to help the user
in constructing his own monitors (`p4_monitor_t').


File: p4.info  Node: Monitor-Building Primitives, Prev: Shared Memory Data Types, Up: Functions for Shared Memory, Next: Some Useful Monitors


Monitor-Building Primitives
---------------------------



The following functions can be used to construct monitors.  A monitor so
constructed has the type `p4_monitor_t'.


     int p4_moninit(m,i)
     p4_monitor_t *m;
     int i;

initializes the monitor pointed to by `m' and gives it `i' queues for
processes to wait on while they are blocked (see `delay').  One queue is
sufficient for most purposes.  The queues are numbered beginning with 0.


     VOID p4_menter(m)
     p4_monitor_t *m;

enter the monitor pointed to by m.  By the definition of a monitor,
access is restricted to a single process in the monitor at a time (if
everybody plays by the rules).


     VOID p4_mexit(m)
     p4_monitor_t *m;

exits the monitor pointed to by m.  You are of course assumed to have
previously entered that monitor.


     VOID p4_mcontinue(m,i)
     p4_monitor_t *m;
     int i;

checks to see if there are any processes blocked on the `i'-th queue of
the monitor `m' and causes one of them to be released for entry to the
monitor if so.  If there are no such processes, the invoking process
simply exits.  Note that a process could have been blocked previously
by invoking the procedure `p4_mdelay'.  The queues are numbered 
beginning with 0.


     VOID p4_mdelay(m,i)
     p4_monitor_t *m;
     int i;

permits a process to delay itself on the `i'-th queue of monitor
`m' if the process wishes to release the monitor, but wants to be
waked up by another process later (via the procedure `p4_mcontinue').
The queues are numbered beginning with 0.



File: p4.info  Node: Some Useful Monitors, Prev: Monitor-Building Primitives, Up: Functions for Shared Memory, Next: 	


Some Useful Monitors
--------------------



In this section we describe some of the specific monitors that are built into
the p4 library.  Each of them has its own pre-defined type, which can be
used to allocate storage for them, which should be in shared memory.
See the `p4/monitors' directory for examples.  A lock is itself a
monitor, with no extra delay queues.


     VOID p4_lock_init(l)
     p4_lock_t *l;

initializes the lock `l'.  Must be used prior to any attempts to lock or
unlock `l'.



     VOID p4_lock(l)
     p4_lock_t *l;

blocks if the lock `l' is already locked, otherwise locks `l'
and proceeds.



     VOID p4_unlock(l)
     p4_lock_t *l;

unlocks the lock `l'.


     VOID p4_getsub(gs,s,max,nprocs)
     p4_getsub_monitor_t *gs;
     int *s,max,nprocs;

is a procedure used to obtain the next value of a shared counter
(subscript).  It takes as its first argument, a pointer to a getsub
monitor that protects the shared counter.  It assigns the current value
of the counter to the integer that s points to, and then increments the
counter by 1.  `p4_getsub_init' initially sets the counter to 0.
When the counter passes the value `max', all `nprocs'
processes are returned the value (-1) once, then the counter is reset to
0 for further use.


     VOID p4_getsubs(gs,s,max,nprocs,stride)
     p4_getsub_monitor_t *gs;
     int *s,max,nprocs,stride;

is like `p4_getsub' except that the counter is increased on each call by
`stride' instead of 1.



     int p4_getsub_init(gs)
     p4_getsub_monitor_t *gs;

initializes the getsub monitor pointed to by `gs'; this initialization
includes assigning a value of 0 to the counter that the monitor
protects.


The standard barrier synchronization pattern is expressed as a monitor.
There can be multiple barrier monitors, and one can wait for only some
of the processes at the barrier if this is desired.



     VOID p4_barrier(b,nprocs)
     p4_barrier_monitor_t *b;
     int nprocs;

causes the executing process to hang until `nprocs' processes execute
a barrier instruction with a pointer to the same barrier monitor `b'
as an argument.


     int p4_barrier_init(b)
     p4_barrier_monitor_t *b;

initializes the barrier monitor `b'; this procedure should be invoked
before you attempt to use the monitor in any operations.

Finally, the `askfor' monitor functions like a general dispatcher of
work. 



     int p4_askfor(af,nprocs,getprob_fxn,problem,reset_fxn)
     p4_askfor_monitor_t *af;
     int nprocs;
     int (*getprob_fxn)();
     VOID *problem;
     int (*reset_fxn)();

requests a new "problem" to work on from the problem pool.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) the number of processes that call this procedure
(with `af') looking for work, (3) a pointer to the user-written procedure
that obtains a problem from the pool, (4) a pointer that is filled in
with the address of a user-defined representation of a problem to be
solved, and (5) a pointer to a user-written procedure to reset when all
problems in the pool are solved, in case the same monitor is re-used
for another set of problems later.  `p4_askfor' returns an integer
indicating whether a problem was successfully obtained or not:

         -1     : program is terminating (some process called p4_progend)
          0     : a problem was obtained and "problem" points to it
          1     : problem solved by exhaustion, i.e. no more problems to get
          n > 1 : a process found a solution and called p4_probend with code n

For a detailed discussion of the "askfor" monitor, see
[lusk-overbeek:p4-book].


     int p4_update(af,putprob_fxn,problem)
     p4_askfor_monitor_t *af;
     int (*putprob_fxn)();
     VOID *problem;

updates the problem pool being managed by the askfor monitor.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) a pointer to the user-written procedure that puts
problems into the pool, and (3) a pointer to a user-defined
representation of a problem to be put in the pool.  `Putprob_fxn'
should return 1 if it did indeed put a new problem into the pool, so
that any delayed processes should wake up and re-examine the pool (this
logic is handled by the `p4_askfor') and 0 if upon entering the
monitor and examining its potential problem together with the data there
it decided not to add a new problem to the pool.  It can be assumed that
the "putprob" logic (defined by `putprob_fxn') is executed inside
the monitor.


     int p4_askfor_init(af)
     p4_askfor_monitor_t *af;

initializes the askfor monitor `af'; this procedure should be invoked
before you attempt to use the monitor in any operations.


     VOID p4_probend(af,code)
     p4_askfor_monitor_t *af;
     int code;

allows the user process to mark a problem as solved early when
several processes are coordinating their activities via an askfor
monitor.  The code is an integer value that will be returned to all
processes when they "askfor" a new sub-problem to work on.


     VOID p4_progend(af)
     p4_askfor_monitor_t *af;

allows a process to cause a return code of (-1) to be returned to all
processes using an askfor monitor.  This would typically be called by
a master process to indicate that no more problems are to be solved
and that all slave processes should terminate.



File: p4.info  Node: Functions for Timing p4 Programs, Prev: Functions for Shared Memory, Up: Top, Next: Functions for Debugging p4 Programs


Functions for Timing p4 Programs
================================



A small number of simple functions are available for accessing various
clocks and timers.  


     int p4_clock()

returns a value in milliseconds.  This is a wall-clock value, usually obtained
from the system via `gettimeofday'.  Also see `p4_ustimer' below.


     p4_usc_time_t p4_ustimer()

returns a wall-clock time value in microseconds.  The precision of this
number depends on the timer installed on the individual machine.  In
some cases the resolution may be no greater than that of `p4_clock()'.
For arithmetic and printing purposes, the type `p4_usc_time_t' is an
unsigned long integer.


     p4_usc_time_t p4_usrollover()

returns the timer value at which a microsecond timer "rolls over".
Since `p4_usc_time_t' is a long integer's worth of microseconds, it is
likely that the timer will roll over (become zero) during even
medium-length runs.



File: p4.info  Node: Functions for Debugging p4 Programs, Prev: Functions for Timing p4 Programs, Up: Top, Next: Miscellaneous Functions


Functions for Debugging p4 Programs
===================================



P4 has a set of routines to aid in producing a printed trace of events, both
user-defined and pre-defined in the p4 system.


     VOID p4_dprintf(fmt, va_alist)
     char *fmt;
     va_dcl

acts just like the standard `printf' except that the print line is
preceded by a value that identifies the process.  This value is
typically the string `pn_u' where `n' represents the
p4-assigned id and `u' represents the unix-id of the process on its
host.  However, there are other forms of this value.  For example, the
big master is represented as `bm_u'.  Also, if a process prints
before it has a p4-assigned id, then its value will be something like
`bm_slave_n_u' or `rm_slave_n_u'.  Typically, it is not
possible for a user program to print anything before being assigned an
id by p4, but the p4 system itself may use this procedure to print
messages from a particular process if it encounters problems getting the
process initialized.


     VOID p4_dprintfl(level, fmt, va_alist)
     int level;
     char *fmt;
     va_dcl

is like `p4_dprintf' except that the first argument is an integer
indicating the debugging level that must be in effect before this
message will print.  A level of 0 will cause the message to always print.
If you run a program with the debug level set to 5 (via command-line
arguments), then all `dprintfl''s with level less than or equal to
that debug level will print. *Note Command-Line Arguments:: for how to
set the debug level at run time.

The debug level can be examined and changed by the user during execution:


     int p4_get_dbg_level()     

returns the current debug level for this process and its cluster.


     VOID p4_set_dbg_level(level)     
     int level;

sets the current debug level for this process and its cluster.
P4 itself is liberally instrumented with `p4_dprintfl''s of level
10 and above, leaving levels 0-9 for the user.  The greater the debug
level of the built-in messages, the greater understanding of p4 needed
by the user to make sense of them.  However, levels as high as 30 may
well be useful to the user trying to debug a p4 program.
Roughly speaking, the following debug levels produce messages about the
indicated events.

     level 10:  created process
                sent message
                received message
     
     level 20:  creating process
                sending message
                receiving message
                process starting
                process exiting
     
     level 30:  waiting for ack
                sending ack
                sent ack
                received ack
                queueing message for later receipt
                queued message for later receipt
     
     level 40:  memory management
                buffer management
     
     level 50:  reading procgroup
                other initialization message exchange
     
     level 60:  send-receive details, especially machine-specific traces
     
     level 70:  listener interactions:
                 creating listener
                 created listener
                 messages from inside listener
     
     level 80:  detailed data structures after initialization
     
     level 90:  detailed tracing of flow thru procedures


For optimum performance, the test of the debug level required by these
messages can be removed at compile time by not commenting out the 
`#define P4_DPRINTFL' line in the `OPTIONS' file.
(*Note Structure of the Distribution Directory::).

The following function is provided to deal with abnormal termination.
It can be called by any process.


     VOID p4_error(string, value)
     char *string;
     int value;

prints `string' as an error message and then forcefully terminates
all co-operating processes and cleans up all shared resources.


     VOID p4_soft_errors(onoff)     
     int onoff;

enables/disables soft errors, returning the previous setting.  The default
is "disabled", which means that certain p4 functions will call
`p4_error' instead of returning -1.


`p4_error' gets control on certain kinds of interrupts.  It is
automatically called for `SIGSEGV', `SIGBUS', and `SIGFPE'
interrupts, to catch user programming errors and clean up, after which it
returns interrupt handling to default mode and returns, so that the user
may obtain a dump.  It also handles `SIGINT' interrupts, in which case
it cleans up and exits.  Finally, it may be called directly by the user, in
which case it cleans up (other p4 processes and IPC's) and exits.


Although `p4_error' is supposed to get rid of all running p4
processes, it can happen that an error is bad enough that p4 processes
are left running.  A primitive aid in finding and killing these
processes is the shell script `kj', which takes a string as an
argument and then kills processes containing that string as part of
their program names.  Currently it only kills processes on the machine
where it is run, but it can be run via `rsh' on remote machines.
There are other useful scripts (e.g. `killipc' and `killp4')
in the `p4/bin' directory to do such things as clean up SYSV IPC 
items that may be left when a program abnormally terminates.  P4 will 
generally cleanup these items if the abnormal termination is a type that 
p4 traps, otherwise the user must do the cleanup.  This is an unfortunate
side-effect of the way that SYSV handles things, it really should be
the OS's function to take care of this.

On many machines it is possible to attach a debugger like `dbx' to a
running process.  This is one way to find out where a hanging process is
stuck. 


File: p4.info  Node: Miscellaneous Functions, Prev: Functions for Debugging p4 Programs, Up: Top, Next: Fortran Interface


Miscellaneous Functions
=======================


In this section are found functions that do seem to fit neatly into any of the
other sections.


     char *p4_version()

returns a string containing the version number of p4 being run.



     VOID p4_get_cluster_masters(numids, ids)
     int *numids, ids[];

This procedure fills in the values of numids and ids.  It obtains the
p4-ids of all "cluster masters" for the program, placing them in the
ids array and placing the number of ids in numids.


     VOID p4_print_avail_buffs()

P4 maintains an array of buffer lists of various sizes, so that it can very
rapidly allocate and deallocate buffers.  You can see the contents of the
buffer pools at any time by calling this procedure.



     VOID p4_set_avail_buff(bufidx,size)
     int bufidx;
     int size;

This procedure is used to set the size of buffers in p4's buffer pools.  The
parameter `bufidx' specifies a particular buffer list, and should be a
number from 0 to 7.  The `size' parameter specifies that buffers up to
that size will be managed by p4 in a particular list.  It is important to
maintain the buffer sizes in increasing order.  The default list of buffer
sizes is {64, 256, 1024, 4096, 16384, 65536, 262144, 1048576}.  This causes
wasted space if you send only one large message, causing the allocation of a
large buffer which is not reused.  Savings in space can be achieved by
adjusting these numbers to correspond with the message sizes of your
application.  If no large messages are sent at all, however, no space is
wasted since the large buffers will never be allocated.  If you send a message
larger that the largest size in this array, p4 will allocate the buffer, and
then free it back to the system as soon as it can.



File: p4.info  Node: Fortran Interface, Prev: Miscellaneous Functions, Up: Top, Next: Faster Startup with the Secure Server


Fortran Interface
=================



The Fortran calls to p4 procedures are analogous to their C counterparts, but
have Fortran-like names.

     #include "p4f.h"
     
     p4init()
     p4crpg()
     p4cleanup()
     p4sendr(type,dest,msg,len,rc)
     p4sendrx(type,dest,msg,len,data_type,rc)
     p4send(type,dest,msg,len,rc)
     p4sendx(type,dest,msg,len,data_type,rc)
     p4recv(type,from,buf,buflen,msglen,rc)
     p4brdcst(type,data,len,rc)
     p4brdcstx(type,data,len,data_type,rc)
     p4probe(type,from,rc)
     p4myclid()
     p4nclids()
     p4getclids(start,end)
     p4myid()
     p4clock()
     p4ustimer()
     p4ntotids()
     p4error(str,val)
     p4softerrs(new,old)
     p4version()
     p4flush()
     p4globop(type,x,nelem,size,op,data_type,rc)
        p4intsumop()
        p4intabsmaxop()
        p4intabsminop()
        p4intmaxop()
        p4intminop()
        p4intmultop()
        p4dblsumop()
        p4dblabsmaxop()
        p4dblabsminop()
        p4dblmaxop()
        p4dblminop()
        p4dblmultop()
        p4fltsumop()
        p4fltabsmaxop()
        p4fltabsminop()
        p4fltmaxop()
        p4fltminop()
        p4fltmultop()


The `data_type' parameter in the above operations shoudl be one of
`P4INT', `P4LNG', `P4FLT', or `P4DBL'.
These symbolic constants are defined in the include file `p4f.h'.


File: p4.info  Node: Faster Startup with the Secure Server, Prev: Fortran Interface, Up: Top, Next: Utilities for Managing a p4 Session


Faster Startup with the Secure Server
=====================================






P4 processes on remote machines are ordinarily created by `rsh'.
For this to work, the user must have permission to create processes on
that machine.  This permission is normally granted either globally by
the system administrator, or locally by the use of `.rhosts' files.
(See the normal unix man pages under `rhosts').

Since `rsh' is relatively slow, p4 provides a way to get things
started faster.  This is accomplished by running the program
`serv_p4' in the background on the remote machine.  When p4 is
creating processes, it will automatically check for the existence of
this server and use it if it is running.  Remote processes typically
start much faster when the server is running.  A disadvantage of using
the server is that output from `p4_dprintf' is lost unless directed
by a command-line argument to a file.  (*Note Command-Line Arguments::.)
When p4 uses `rsh', the remote process's `stdout' is sent
back to the `stdout' of the parent (the p4 master process).
We have not yet tested this server on all of the machines that we support.
Thus far, we have tested it somewhat on the SYMMETRY, SUN, DEC500, and
SGI.  We believe that it will work on many other machines, but have not
yet verified it on all machines.

An invocation of a set of servers is (currently) associated with a specific
port number.  This way multiple users can each be running multiple server
networks without mutual interference, provided each network of servers is
started with a different port number. 

To start the secrue server on a machine one can do

     serv_p4 -d -p <num>


where `<num>' is a port number to be associated with a network of
servers.  If the `-p' option is omitted, the server will pick an unused
port number and report

     Listening on <num>.


Then p4 programs to use this network should be started with 

     -ssport <num>


The p4 application must also be listed in the user's `.p4apps' file in
his home directory.  This file should be readable only by the user, and should
contain the full path names of programs that the user wishes to be startable
by the p4 server.

When a p4 master process tries to start a slave process on a remote machine,
it will first attempt to do it via the server.  If it cannot do so for any
reason (no server running, port number mismatch, or program not found in
`.p4apps' file), then it tries to do so with the remote shell command.

Note that the server is used only to start processes; it plays no role in a p4
computation once the slave processes have been initiated.  Rather, a temporary
process, called the {listener}, is spawned to manage connection requests
that occur during the execution of a p4 program.  Neither the server nor the
listener consumes any significant amount of CPU time.

There is further discussion of installation options for the servers in
the `README' file in the `p4/servers' subdirectory.


File: p4.info  Node: Utilities for Managing a p4 Session, Prev: Faster Startup with the Secure Server, Up: Top, Next: Creating Logfiles for Upshot


Utilities for Managing a p4 Session
===================================




A number of useful utilities can be found in the `bin' subdirectory.
These can be used to start and stop server processes based on the contents of
a file of machines one regularly uses, to kill runaway p4 processes in the
unlikely case that they cannot or do not terminate automatically when one
processes ends abnormally or interrupted from the keyboard, and to merge
logfiles created for the use of `upshot' *Note Creating Logfiles for Upshot::. 


File: p4.info  Node: Creating Logfiles for Upshot, Prev: Utilities for Managing a p4 Session, Up: Top, Next: Machine-Specific Notes


Creating Logfiles for Upshot
============================






P4 is distributed with a set of routines for creating logfiles (see
`README' in the `p4/alog' directory.  The resulting logfiles
can be examined by `upshot', distributed separately.  For details
about `upshot', see [herrarte-lusk:upshot].  

The `p4/alog' directory contains a package (ALOG) for creating logs of
time-stamped events, that is of general utility, outside of p4.  The
timestamps are obtained from various microsecond-level resolution timers on
various machines.  The portable microsecond timing package is contained in the
`usc' subdirectory.  It is used by the ALOG package as well as by the
`p4_ustimer' function in p4.  Similarly, the ALOG package can be used
independently of p4 and `upshot'. Its logfiles were designed to be read
and displayed by upshot, but other display packages can be used as well.

* Menu:

* User-Specified Events::
* Examining Log Files with Upshot::
* Automatic Logging of p4 Events::



File: p4.info  Node: User-Specified Events, Prev: 	, Up: Creating Logfiles for Upshot, Next: Examining Log Files with Upshot


User-Specified Events
---------------------



The ALOG package consists of a set of macros that can be used to instrument a
C program and a set of functions that can be used to instrument a Fortran
program.  We will focus here primarily on the use of the C interface, which
contains more functionality.

The macros that can be used to instrument a program are as follows (from the 
file `README_ALOG' in the `alog' directory):


     ALOG_SETUP(pid,flag):	
       pid  -  (integer) process id of callee
       flag -  (integer) either ALOG_WRAP or ALOG_TRUNCATE  

This macro initializes the tracing area for a slave process and must be called
once before any event is logged.  If the value of `flag' is set to
`ALOG_WRAP', then in the event of no more space for logging events the
system will only report the latest n events.  If `flag' is set to
`ALOG_TRUNCATE' the system will stop logging events as soon as there is
no more memory for the events to be logged.


     ALOG_MASTER(pid,flag):
       pid  -  (integer) process id of the callee
       flag -  (integer) either 0 or 1  (see above)

`ALOG_SETUP' with the difference that this macro should be referenced by
the master process only.


     ALOG_DEFINE(event,strdef,format):
       event  - (integer) id of event being defined
       strdef - (string) description of 'event'
       format - (string) control string in "printf" format to

This macro puts an event definition code into the logfile.


     ALOG_LOG(pid,event,intdata,strdata):
       pid     - (integer) process id of callee
       event   - (integer) event id to be logged
       intdata - (integer) any integer data for this event
       strdata - (string) any string data (can be the null string)

This macro provides the event logging service.


     ALOG_OUTPUT
       no parameters  

This macro dumps the events logged into a log file with the name
`alogfile.pxx' where `xx' is the logical PID of the callee process.
The log file is created in the current directory unless specified otherwise
through the macro ALOG_SETDIR.


     ALOG_SETDIR(dir)
       dir - (string) directory where log file is created

This macro sets the output directory for the log file.  The default directory
for the creation of the log file is the current directory of the process.  If
used, then this macro MUST be invoked before `ALOG_MASTER/ALOG_SETUP'.


     ALOG_STATUS(status):
       status - (integer) either ALOG_ON or ALOG_OFF


This macro controls the logging status of `ALOG' as follows.  Setting
`status' to `ALOG_ON' enables logging until it is turned off.
Setting `status' to `ALOG_OFF' disables logging until it is turned
on again.  Logging is enabled at the outset by default.



     ALOG_ENABLE
       no parameters


This macro enables event logging; same as calling `ALOG_STATUS(ALOG_ON)'.


     ALOG_DISABLE
       no parameters

This macro disables event logging; same as calling
`ALOG_STATUS(ALOG_OFF)'.

The sample program `gridlog.shmem.c' in the `monitors' subdirectory
contains an example of a program instrumented with ALOG statements.  The macro
definitions for ALOG are included when you include `#include "p4"' in
your program.  If the line `#define ALOG_TRACE' is not included before
the `#include "p4"', these macros will generate no code.  Thus it is
easy to effectively de-instrument the code by recompiling, and there is no
need to protect each ALOG statement with an `#ifdef'.
 
When an ALOG-instrumented program is run, it will produce one logfile for each
process.  The files will be named `alogfile.p0', `alogfile.p1',
.  These files need to be merged into a single file with the events
sored by timestamp.  This is accomplished with the program `mergelogs',
found in the `bin' subdirectory.  To merge the logfiles, do 

     mergelogs alogfile.p* > myprog.log
     rm alogfile.p*


The resulting logfile can be examined by upshot or some other logfile
examination facility.  See [herrarte-lusk:upshot] for details of the
logfile format.

On networks of workstations and some distributed memory machisnes, the
microsecond timers on the various processors are synchronized.  To produce a
usable merged logfile, the `adjlogs' program, also found in the
`bin' directory, can be used to adjust the timestamps for offset and
drift before they are merged.  For this to work, synchronization events must
be placed in the logfiles by an `ALOG_LOG' statement.  The event type
is then passed to `adjlogs', which aligns the timestamps, based on the
timestamps of the synchonization events.  The call to `adjlogs' looks
like this, where `<n>' is the type of the synchronization event.

     adjlogs -e <n>


Both `mergelogs' and `adjlogs' are less portable than the other p4
code;  you might want to run them on a workstation such as a Sun.


File: p4.info  Node: Examining Log Files with Upshot, Prev: User-Specified Events, Up: Creating Logfiles for Upshot, Next: Automatic Logging of p4 Events


Examining Log Files with Upshot
-------------------------------




`Upshot' is not part of the p4 distribution, but can be obtained from the
same anonymous `ftp' location as p4.  Take the file `upshot.tar.Z'
from the directory `pub/p4' on `info.mcs.anl.gov'.  The distribution
contains all necessary documentation on how to install and run `upshot'.
It is an X-window program that runs on most workstations.  There is no need
for a parallel macchine to be involved, once the log files have been obtained.

`Upshot' produces the most interesting displays when certain events
(not necessarily all) are defined to be the entry and exit events for certain
{states} and then colors are associated with the states.  This association
is reflected in a {statefile} with a format like the following:

     1 1 2 red   asking
     2 3 4 blue  working
     3 5 6 green updating


This statefile describes three states.  State 1 is defined to be between
events 1 and 2.  `Upshot' will color it red and label it "asking".


File: p4.info  Node: Automatic Logging of p4 Events, Prev: Examining Log Files with Upshot, Up: Creating Logfiles for Upshot, Next: 	


Automatic Logging of p4 Events
------------------------------



We have found that the most useful events to log and study are those
identified by the user and specified in his program.  That way he can control
the number of events to be logged and the grain size of the states that are
represented.  

In some cases, however, one wants to study the details of the internal
operation of a p4 application, or get some idea of the behavior on one's
program without going to the trouble of instrumenting it himself.  To this
end, p4 itself is instrumented with ALOG statements, although by default they
are inactive.  To get automatic logging of p4 events (including sending and
receiving of each message) one needs first to link to a version of the p4
library that has been compiled with the line `#define ALOG_TRACE'
uncommented out in the `OPTIONS' file, and secondly, to run with
`-p4log' on the command line.


File: p4.info  Node: Machine-Specific Notes, Prev: Creating Logfiles for Upshot, Up: Top, Next: Some Common Problems and their Solutions


Machine-Specific Notes
======================


     SUN
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     HP
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not tested (not avail on our test machine).
     
     DEC5000
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     RS6000
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     IBM3090
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not supported due to absence of iargc/getarg.
         (3)  There are multiply defined macros in include/rpc/rpc.h.  IBM
              is fixing this in a later OS release.  Meanwhile make your own
              copy and the fix the problem yourself.
         
     TITAN
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not supported due to problems with getting args.
     
     SGI
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     NEXT
         (1)  Fortran not supported due to absence of iargc/getarg.
     
     FX2800/FX2800_SWITCH
         (1)  Alliant's switch code not yet ensuring messages 
              remain ordered.  p4 currently discovers the switch port for
              the machine it is running on by invoking the internal
              procedure getswport.  This procedure must be customized to
              the installation.
           
     FX8
         (1)  You might need to add MFLAGS = -i to the Makefile
     
     KSR
         (1)  The latest version of the OS produces a link-time error for
              Fortran programs.
     
     IPSC860
         (1)  the script "runcube" (in the messages directory) may 
              be useful
     	 
     DELTA
         (1)  the script "rundelta" (in the messages directory) may 
              be useful
     
     BALANCE
         (1)  Fortran not supported.
     
     SYMMETRY/SYMMETRY_PTX
         (1)  -Z compiler option may be used to control shmalloc/malloc split
         (2)  shared memory message passing not supported in Fortran
     
     TC_2000/TC_2000_TCMP
         (1)  TCMP port not yet complete.
         (2)  For shared-memory execution, one must use `cluster ...' to
              obtain a private cluster for execution



File: p4.info  Node: Some Common Problems and their Solutions, Prev: Machine-Specific Notes, Up: Top, Next: Concept Index


Some Common Problems and their Solutions
========================================




Our attempt with this manual has been to prevent you from having difficulties.
Experience shows that certain common progblems recur, however.  In this
section we hope to address some of these problems.

"Permission Denied."     
      p4 slave processes are started by forks (for
       slaves in the same shared-memory cluster), by the server, or by the remote
       shell command.  If the server is running on the target machine then that
       must be configured to allow remote processes to be started.  To test whether
       this is your problem, try
              rsh target.machine date
     
       If you still get the "Permission denied." message, then the problem has
       nothing to do with p4.  See `hosts.equiv' or `.rhosts' in the
       system man pages.
"More processes than message queues"     
       Under the default configuration
       of p4, uniprocessors, such as most workstations, cannot have multiple
       process sharing memory.  Thus your procgroup file for a workstation network
       should always look like
              local 0
              machine1 1 pathname
              machine2 1 pathname
              machine3 1 pathname
                 .
                 .
     
       The "local" means "only the master on the startup machine; no local
       slaves sharing memory".
     
     
       It is possible, at some cost in message-passing efficiency, to have a
       cluster of processes sharing memory on a workstation, but in this case p4
       must have been installed with the `SYSV_IPC' option set in the
       `OPTIONS' file.  The cost is that a process waiting for a message must
       spin between checking for a message arriving on a socket and a message
       arriving through shared memory. 
     
"p4error: local is not first entry in procgroup"     
       the first line of
       the procgroup file must be the "local" entry, specifying the number of
       slaves that will be run on the master machine in addition to the master
       process. 
     
"gethostbyname failed 100 times"     
      Check for an invalid machine name in
       the procgroup file.
"pgmpathname: Command not found"     
       P4 tried to start the program
       with the given name on a remote machine and the program did not exist.
       Verify the full path name of the program.
program hangs     
       You may have failed to initialize the `type' and
       `from' fields before a `p4_recv'.  You might have used
       `p4_sendr' between two processes at the same time, which will deadlock
       if you think about it, or even if you don't.  Use `p4_send' instead.
program hangs or has bad data in received message     
       You might have
       failed to set the pointer to the incoming buffer to NULL, or to have
       specifically allocated a buffer with `p4_msg_alloc', before a
       `p4_recv'.
program ignores command-line arguments     
       You might have passed
       `argc' instead of `&argc' to `p4_initenv'.
program runs out of memory     
       You may need to call `p4_msg_free'
       after each `p4_recv', or reuse buffers by pre-allocating them.
     
     



File: p4.info  Node: Concept Index, Prev: Some Common Problems and their Solutions, Up: Top, Next: Function Index


Concept Index
=============


* Menu:

* allocating buffers: Explicit Sending and Receiving of Messages.
* askfor monitor: Some Useful Monitors.
* automatic logging: Automatic Logging of p4 Events.
* barrier: Global Operations.
* barrier: Some Useful Monitors.
* buffer pool management: Miscellaneous Functions.
* clocks: Functions for Timing p4 Programs.
* cluster management: Functions for Cluster Management.
* command-line arguments: Command-Line Arguments.
* data types for monitors: Shared Memory Data Types.
* deallocating buffers: A More Complicated Example.
* deallocating buffers: Explicit Sending and Receiving of Messages.
* debugging: Functions for Debugging p4 Programs.
* directory structure: Structure of the Distribution Directory.
* documentation: Installing the Documentation.
* emacs Info: Installing the Documentation.
* examples: Examples included with the Distribution.
* faster startup: Faster Startup with the Secure Server.
* Fortran interface: Fortran Interface.
* getting started: Getting Started.
* global operations: Global Operations.
* installation: Installing p4.
* interrupts: Functions for Debugging p4 Programs.
* IPC: Some Common Problems and their Solutions.
* lock: Some Useful Monitors.
* logging: Creating Logfiles for Upshot.
* machines: Installing the p4 System.
* man pages: Installing the Documentation.
* message-passing functions: Functions for Message Passing.
* monitor data types: Shared Memory Data Types.
* monitor primitives: Monitor-Building Primitives.
* monitors: Some Useful Monitors.
* online help: Installing the Documentation.
* OPTIONS: Structure of the Distribution Directory.
* permission denied: Some Common Problems and their Solutions.
* permissions: Faster Startup with the Secure Server.
* problems: Some Common Problems and their Solutions.
* process management: Functions for Process Management.
* procgroup files: Specifying Processes in the Procgroup File.
* receiving messages: Explicit Sending and Receiving of Messages.
* return codes: Return Codes from p4 Functions.
* rhosts: Faster Startup with the Secure Server.
* runaway processes: Functions for Debugging p4 Programs.
* sending messages: Explicit Sending and Receiving of Messages.
* servers: Faster Startup with the Secure Server.
* servers: Utilities for Managing a p4 Session.
* shared memory data types: Shared Memory Data Types.
* shared memory functions: Functions for Shared Memory.
* slave subroutine: A Minimal Example.
* statefile: Examining Log Files with Upshot.
* testing: Installing the p4 System.
* timestamps: Creating Logfiles for Upshot.
* timing: Functions for Timing p4 Programs.
* tracefiles: Creating Logfiles for Upshot.
* unlock: Some Useful Monitors.
* upshot: Creating Logfiles for Upshot.
* upshot: Examining Log Files with Upshot.
* user events: User-Specified Events.
* utilities: Utilities for Managing a p4 Session.




File: p4.info  Node: Function Index, Prev: Concept Index, Up: Top, Next: 	


Function Index
==============



* Menu:

* ALOG_DEFINE: User-Specified Events.
* ALOG_DISABLE: User-Specified Events.
* ALOG_ENABLE: User-Specified Events.
* ALOG_LOG: User-Specified Events.
* ALOG_MASTER: User-Specified Events.
* ALOG_OUTPUT: User-Specified Events.
* ALOG_SETDIR: User-Specified Events.
* ALOG_SETUP: User-Specified Events.
* ALOG_STATUS: User-Specified Events.
* p4_alloc_procgroup: Functions for Process Management.
* p4_am_i_cluster_master: Functions for Cluster Management.
* p4_askfor: Some Useful Monitors.
* p4_askfor_init: Some Useful Monitors.
* p4_barrier: Some Useful Monitors.
* p4_barrier_init: Some Useful Monitors.
* p4_broadcast: Global Operations.
* p4_broadcastx: Global Operations.
* p4_clock: Functions for Timing p4 Programs.
* p4_create: Functions for Process Management.
* p4_create_procgroup: Functions for Process Management.
* p4_dprintf: Functions for Debugging p4 Programs.
* p4_dprintfl: Functions for Debugging p4 Programs.
* p4_error: Functions for Debugging p4 Programs.
* p4_free: Managing Shared and Local Memory.
* p4_get_cluster_ids: Functions for Cluster Management.
* p4_get_cluster_masters: Miscellaneous Functions.
* p4_get_dbg_level: Functions for Debugging p4 Programs.
* p4_get_my_cluster_id: Functions for Cluster Management.
* p4_get_my_id: Functions for Process Management.
* p4_getsub: Some Useful Monitors.
* p4_getsub_init: Some Useful Monitors.
* p4_getsubs: Some Useful Monitors.
* p4_global_barrier: Global Operations.
* p4_global_op: Global Operations.
* p4_initenv: Functions for Process Management.
* p4_lock: Some Useful Monitors.
* p4_lock_init: Some Useful Monitors.
* p4_malloc: Managing Shared and Local Memory.
* p4_mcontinue: Monitor-Building Primitives.
* p4_mdelay: Monitor-Building Primitives.
* p4_menter: Monitor-Building Primitives.
* p4_messages_available: Explicit Sending and Receiving of Messages.
* p4_mexit: Monitor-Building Primitives.
* p4_moninit: Monitor-Building Primitives.
* p4_msg_alloc: Explicit Sending and Receiving of Messages.
* p4_msg_free: Explicit Sending and Receiving of Messages.
* p4_num_cluster_ids: Functions for Cluster Management.
* p4_num_total_ids: Functions for Process Management.
* p4_num_total_slaves: Functions for Process Management.
* p4_print_avail_buffs: Miscellaneous Functions.
* p4_probend: Some Useful Monitors.
* p4_progend: Some Useful Monitors.
* p4_recv: Explicit Sending and Receiving of Messages.
* p4_send: Explicit Sending and Receiving of Messages.
* p4_sendb: Explicit Sending and Receiving of Messages.
* p4_sendbr: Explicit Sending and Receiving of Messages.
* p4_sendbrx: Explicit Sending and Receiving of Messages.
* p4_sendbx: Explicit Sending and Receiving of Messages.
* p4_sendr: Explicit Sending and Receiving of Messages.
* p4_sendrx: Explicit Sending and Receiving of Messages.
* p4_sendx: Explicit Sending and Receiving of Messages.
* p4_set_avail_buff: Miscellaneous Functions.
* p4_set_dbg_level: Functions for Debugging p4 Programs.
* p4_shfree: Managing Shared and Local Memory.
* p4_shmalloc: Managing Shared and Local Memory.
* p4_soft_errors: Functions for Debugging p4 Programs.
* p4_startup: Functions for Process Management.
* p4_unlock: Some Useful Monitors.
* p4_update: Some Useful Monitors.
* p4_usrollover: Functions for Timing p4 Programs.
* p4_ustimer: Functions for Timing p4 Programs.
* p4_version: Miscellaneous Functions.
* p4_wait_for_end: Functions for Process Management.





Tag table:
Node: Top94
Node: Introduction920
Node: Structure of the Distribution Directory4274
Node: Installing p46105
Node: Installing the p4 System6906
Node: Installing the Documentation9890
Node: Examples included with the Distribution11100
Node: Getting Started11908
Node: A Message-Passing Example12243
Node: Analysis of the Program13437
Node: Specifying Processes in the Procgroup File15976
Node: Developing a Simple p4 Program20514
Node: A Minimal Example21050
Node: A More Complicated Example24028
Node: Command-Line Arguments30347
Node: The p4 Function Library31144
Node: Overview of the Library31412
Node: Return Codes from p4 Functions31989
Node: p4 Functions for Managing Processes and Clusters32435
Node: Functions for Process Management33356
Node: Functions for Cluster Management38492
Node: Functions for Message Passing40224
Node: Explicit Sending and Receiving of Messages41038
Node: Global Operations45306
Node: Functions for Shared Memory47866
Node: Managing Shared and Local Memory48199
Node: Shared Memory Data Types48975
Node: Monitor-Building Primitives49652
Node: Some Useful Monitors51353
Node: Functions for Timing p4 Programs56823
Node: Functions for Debugging p4 Programs57897
Node: Miscellaneous Functions63665
Node: Fortran Interface65550
Node: Faster Startup with the Secure Server67024
Node: Utilities for Managing a p4 Session70122
Node: Creating Logfiles for Upshot70799
Node: User-Specified Events71930
Node: Examining Log Files with Upshot76842
Node: Automatic Logging of p4 Events78011
Node: Machine-Specific Notes79062
Node: Some Common Problems and their Solutions81601
Node: Concept Index84958
Node: Function Index87958

End tag table
SHAR_EOF
if test -f 'p4.txt'
then
	echo shar: over-writing existing file "'p4.txt'"
fi
cat << \SHAR_EOF > 'p4.txt'




	      User's Guide to the p4 Parallel Programming System

			 Ralph Butler and Ewing Lusk

			 Argonne National Laboratory



Introduction
============


P4 is a library of macros and subroutines developed at Argonne National
Laboratory for programming a variety of parallel machines.  Its
predecessor was the m4-based "Argonne macros" system described in the
Holt, Rinehart, and Winston book {Portable Programs for Parallel
Processors}, by Lusk, Overbeek, et al., from which p4 takes its
name[lusk-overbeek:p4-book].  The current p4 system maintains the
same basic computational models described there (monitors for the
shared-memory model, message-passing for the distributed-memory model,
and support for combining the two models) while significantly increasing
ease and flexibility of use. *Note Getting Started:: for a simple example.

P4 is intended to be portable, simple to install and use, and efficient.  It
can be used to program networks of workstations, advanced parallel
supercomputers like the Intel Touchstone Delta and the Alliant Campus
HiPPI-based system, and single shared-memory multiprocessors.  It has
currently been installed on the following list of machines: Sequent Symmetry
(Dynix and PTX), Encore Multimax, Alliant FX/8, FX/800, and FX/2800, Cray
X/MP, Sun, NeXT, DEC, Silicon Graphics, HP, and IBM RS6000 workstations,
Stardent Titan, BBN GP-1000 and TC-2000, Intel IPSC/860, Intel Touchstone
Delta, Alliant Campus, and Thinking Machines' CM-5.  It will soon be ported to
to the Intel Paragon.  It is not difficult to port to new systems.  Although
p4 tries to be completely portable, there are a small number of
specific exceptions (*Note Machine-Specific Notes::) that may need to be taken
into account on a given machine.

You can obtain the complete distribution of p4 by anonymous ftp from
`info.mcs.anl.gov'.  Take the file `p4.tar.Z' from the
directory `pub/p4'.  The distribution contains all source code,
installation instructions, this reference manual, and a collection of
examples in both C and Fortran. `Alog' is included in the
distribution with p4.  The file `upshot.tar.Z' contains display
facilities that can be used with p4 and other systems.

To ask questions about p4, report bugs, contribute examples, etc., you can
send mail to `p4@mcs.anl.gov'.

The current release is version 1.2.  You can check which version of the source
code you have by looking at the file `lib/p4_patchlevel.h' in the
distribution.  You can check which version of the object code you have linked
to by running any p4 program with the command-line option `-p4version'
(*Note Command-Line Arguments::).

Salient features of the current release of p4 include: 
   * `xdr' support for heterogeneous networks
   * manual enhanced and converted to latex format
   * Emacs info version of the manual for on-line help
   * SYSV IPC support added for several machines (for shared-memory
       multiprocessing on workstations that support multiple processors)
   * instrumentation added for automatic logging/tracing
   * automatic or user control of message-passing/buffer-management
   * high-resolution clock support for several machines
   * error/interrupt handling
   * an optional secure server for quick startup


A useful companion system is the `alog/upshot' logging and X-based trace
examination facility.  (*Note Creating Logfiles for Upshot::.)



Structure of the Distribution Directory
=======================================



The p4 source code distribution contains the following files and
subdirectories:


Makefile     
      The makefile for making the p4 system, doing the installation,
       and making makefiles for user applications.
OPTIONS     
      A file controlling various compile-time options, such as
       whether System V shared-memory operations are to be enabled, whether system
       debug message printing is to be enabled, and whether automatic
       instrumentation of p4 operations for the `upshot' logging and tracing
       program is to be done.
README     
      General instructions.
alog     
      Source code for the ALOG tracing package.
bin     
      Scripts for starting and killing servers, killing runaway p4
       processes, merging `upshot' logfiles, and other useful utilities.
contrib     
      Examples contributed by p4 users.
contrib_f     
      Fortran examples contributed by users.
doc     
      The man page, together with this manual and supporting files.
include     
      The include directory for making p4 applications.  Most of
       these are (hard) links into the `lib' directory.
lib     
      The source code for the p4 system.
lib_f     
      The Fortran interface for p4.  
messages     
      A basic set of message-passing examples in C.
messages_f     
      A basic set of message-passing examples in Fortran.
monitors     
      A basic set of shared-memory examples in C.
servers     
      The secure and insecure servers.
usc     
      The portable microsecond clock routines.
util     
      Assorted supporting files, particularly for making the p4
       distribution. 




Installing p4
=============



In this section we describe how to install the p4 library, either for
your own personal use or for the use of everyone at your site.  In the
first case you do not need any super-user privileges.  In the second
case, you may or may not, depending on how things are configured at
your site.  We also describe how to install and run the examples that
come with p4, the online help system (this manual as an emacs info-file)
and how to build a working directory for your own programs yet share
the installed copy of p4 with other users.



Installing the p4 System
------------------------


To build p4, position yourself in the `p4' directory and type:
     make all MACHINE=<machine>

where <machine> is one of the machine names listed in
`p4/util/machines', currently:


SUN     
      Sun-3, Sun386i, Sparc-1, or Sparc-2 workstations
     
HP     
      HP workstations
     
DEC5000     
      Dec 5000 workstations
     
NEXT     
      68030- or 68040-based NeXT workstations
     
RS6000     
      IBM RS 6000 series workstations
     
IBM3090     
      IBM 3090 running IBM's version of UNIX, AIX
     
BALANCE     
      Sequent Symmetry shared-memory multiprocessor
     
SYMMETRY     
      Sequent Symmetry shared-memory multiprocessor
     
SYMMETRY_PTX     
      Sequent Symmetry shared-memory multiprocessor PTX OS
     
MULTIMAX     
      Encore Multimax shared-memory multiprocessor
     
GP_1000     
      BBN GP-1000
     
TC_2000     
      BBN TC-2000
     
TC_2000_TCMP     
      BBN TC-2000 with the TCMP message-passing library
     
IPSC860     
      Intel IPSC/860 (nodes only)
     
IPSC860_SOCKETS     
      Intel IPSC/860 with socket libraries on the nodes
     
DELTA     
      Intel DELTA 
     
TITAN     
      Stardent Titan
     
SGI     
      Silicon Graphics workstations
     
CRAY     
      Cray X/MP
     
FX8     
      Alliant FX/8
     
FX2800     
      Alliant FX/2800 or FX/800
     
FX2800_SWITCH     
      Alliant FX/2800 or FX/800, with CAMPUS HiPPI switch
     
KSR     
      Kendall Square KSR-1
     
CM-5     
      Thinking Machines' CM-5






For example:
     make all MACHINE=SYMMETRY

The `all' is optional, for example
     make MACHINE=SEQUENT

This will create a machine-dependent `Makefile' in each subdirectory,
make the p4 library, and compile and link a subset of the examples.

To add a new machine type, or to change the characteristic parameters
associated with an existing one, you can edit the file 
`p4/util/defs.all'.

To save disk space, various intermediate object files can be removed with
     make clean

The system can be restored to its original, machine-independent state with
     make realclean

Note that this removes the machine-dependent Makefiles in each directory, so
the operation is not idempotent.

It is also possible to install (or clean) only some of the directories:
     make all MACHINE=SUN DIRS=messages
     make clean DIRS='monitors messages'

To install only the Makefiles in all subdirectories, use:
     make makefiles MACHINE=<machine>

directory everything that is needed to compile and link p4 programs, 
do:

     make install INSTALLDIR=<dir>

minimal set of directories, copy the relevant `.a' and `.h' files
into it, and test the installation by mking a small set of examples.

*Note Getting Started:: for instructions on how to run some example
programs after you have installed p4.


Installing the Documentation
----------------------------



The directory `p4/doc' contains this manual as well as files that
require installation.  This manual was prepared with the
`latexinfo' package from GNU emacs; thus it can be made available online
through `info'.  The files in `p4/doc' are:

p4.tex     
      the latex source for this manual, which uses the latexinfo style
     
latexinfo.sty     
      the sytle file needed to latex this manual
     
p4.info     
      the `info' version of this manual, ready to be put
     into the directory where `info' files are kept at your site.  Check
     the value of your `emacs' variable INFO-DIRECTORY.
     
p4.txt     
      plain ascii text of the manual, in case nothing else works.
     
p4.1     
      unix man page for the p4 library
     
p4f.1     
      unix man page for the Fortran interface to p4
     


The Postscript version of this manual is available by anonymous `ftp'
from `info.mcs.anl.gov', in the directory `pub/p4'.  The file to
get (in binary mode) is `p4man.ps.Z'.



Examples included with the Distribution
---------------------------------------



A good way to see how various p4 functions are used is to look at the example
programs included in the distribution.  The `p4/monitors' directory
contains shared-memory examples written in C that use monitors, including one
instrumented with ALOG.  The `p4/messages' subdirectory contains
message-passing examples written in C.  The programs in `p4/messages_f'
are Fortran message-passing examples, and the `p4/contrib' and
`p4/contrib_f' directories contain a number of miscellaneous examples
contributed by users.  In each directory there is a `README' that
describes the individual examples.



Getting Started
===============



The easiest way to get started with p4 is to play with some of the
sample programs provided with the system.



A Message-Passing Example
-------------------------


We will begin with a message-passing example in the sub-directory named
`p4/messages'.  The code for the program is in the files `sr_test.c'
and `sr_user.h'.


 
Program Description
-------------------


As the name implies, this program is an example of p4's send/receive
functionality.  Briefly, it is a simple program that runs a master
process and some slave processes.  The master and the set of slaves
form a ring of processes in which the master reads a message from stdin
and sends a copy of the message to the first slave, which passes it on;
the last slave passes the message back to the master.  If the master
receives an undamaged copy of the message, it assumes that all went
well, and reads another message.  Note that the ring of processes is a
logical structure in which each process assumes that its
predecessor in the ring is the process with the next lower id, and
its successor is the process with the next higher id.  The master
has id 0 (zero) and has the process with the largest id as its
predecessor.




Analysis of the Program
-----------------------


The first executable p4 statement in a program should be:
     p4_initenv(&argc,argv); 

This initializes the p4 system and allows p4 to extract any command
line arguments passed to it, e.g. debugging parameters.  

Similarly, the last executable p4 statement in a program should be:
     p4_wait_for_end(); 

This waits for termination of p4 processes and performs some cleanup
operations.

The procedure `p4_get_my_id' returns the unique integer id assigned
to the calling process by p4.

The statement:
     p4_create_procgroup();

reads a procgroup file that the user builds and creates the set of
slaves described in that file.  Obviously this statement must be
executed before any slaves can be assumed to exist.  This procedure
is the method you must use to create processes that do message-passing.

The procedure `p4_clock' returns an integer that represents
wall-clock time in milliseconds.  It is typically used to retrieve the
time before and after some work, the difference representing the time to
do that work.  Note that there is also a `p4_ustimer' that is useful on
those machines that support a microsecond timer.

The procedures `p4_send' and `p4_sendr' are two of several 
p4 procedures that are available for sending messages to other processes.
They take as arguments the message type, the id of the "to" process,
the address of the message, and the message length.

The procedure `p4_recv' receives a message from another process and
sets the values of all four parameters.  `P4_recv' will automatically
retrieve a buffer in which to place a received message, thus
`p4_msg_free' may be called to free that buffer when it is no longer
needed.

The procedure `p4_num_total_slaves' is one of several procedures that
the user can invoke to determine information about the current execution.

To run this program, you need to create a procgroup file that describes
where all slave processes are to be executed 
(*Note Specifying Processes in the Procgroup File::).  We will assume that
you have an example procgroup file (named `procgroup') in the 
`p4/messages' directory, and can run `sr_test' by merely typing:
     sr_test

If the procgroup file is elsewhere, then you must type:
     sr_test -pg  {pathname_of_procgroup_file}


Another example that is made by default is the program `systest'.  It
tests a number of the message-passing features of p4.



Specifying Processes in the Procgroup File
==========================================



The procgroup file is the only portion of the interface that is very
likely to change through multiple versions of p4.  As new architectures
are supported, it is hoped that we can merely alter the procgroup file
format to reflect any new features.  (Of course new procedure calls may 
also be required, but existing procedure calls will remain unchanged 
when possible).

The current format of a procgroup file is as follows:
     local n [full_path_name] [loginname]
     remote_machine n full_path_name [loginname]
     .
     .
     .


On cube and mesh architectures, the program is started via some
special command executed from the host machine.  In such cases, the
procgroup file name can be specified to the special command line along
with the program name (see for example the `runcube' and `rundelta'
shell scripts in the `p4/messages' subdirectory).  In those cases 
where no special command is required, no special handling is required 
for the procgroup filename.

The first line of a procgroup file must be "local n" where n is the
number of slave processes that are in the same cluster as the master.
The full path name on the "local" line is ignored on machines other
than cube and mesh machines.  The subsequent lines contain either
three or four fields:
  1. the name of a remote machine on which slave processes are to be created.
  2. the number of slaves that are to be created on that machine, 
     i.e. be in the same cluster (note that on machines that support it, 
     the processes in a cluster will share memory)
  3. the full path name of the executable slave program
  4. optionally, the user login name on the remote machine, if different from 
     that on the host machine.


As an example, let's assume that you have a network of three Sun 
workstations named sun1, sun2, and sun3.  We will also assume that you 
are working on sun1 and plan to run a master process there.  
If you would like to run one process on each of the other Suns, then you 
might code a procgroup file that looks like:

         # start one slave on each of sun2 and sun3
         local 0 
         sun2  1  /home/mylogin/p4pgms/sr_test
         sun3  1  /home/mylogin/p4pgms/sr_test


Lines beginning with `#' are comments.

Next, let's assume that you have a Sequent Symmetry (named symm) and an
Encore Multimax (named mmax).  We will also assume that you are working
on symm, and plan to run the master there.  If you would like to run
two processes on symm (in addition to the master) and two on mmax, then 
you might code a procgroup file that looks like:

         local 2 
         mmax  2  /mmaxfs/mylogin/p4pgms/sr_test


P4 also permits you to treat the symmetry as a remote machine even when 
you are running the master there.  Thus, you might code a procgroup file 
as follows:

         local 2 
         symm  2  /symmfs/mylogin/p4pgms/sr_test
         mmax  2  /mmaxfs/mylogin/p4pgms/sr_test


In this example, there are seven processes running.  Five of the
processes are on symm, including the master.  Two of the processes on
symm are in the master's procgroup and two are running in a separate
procgroup as if they were on a separate machine.  Of course, the last
two are running on mmax.

Some notes about the contents of the procgroup file should be made at
this point.  First, the value of `n' on the local line can be zero,
i.e. the master may have no local slaves.  Second, the local machine
may be treated as if it is a remote machine by merely entering it in
some line as a remote machine.  Third, a single machine may be treated
as multiple remote machines by having the same remote machine name
entered on multiple lines in the procgroup file.  Fourth, if a single
machine is listed multiple times, those processes specified on each
line form a single cluster (share memory).  Fifth, the cluster size
specified for a uniprocessor should be 1, because all slaves in a
cluster are assumed to run in parallel and to share memory.

We refer to the original (master) process as the "big master".  The
first process created in each cluster is the "remote master" or the
"cluster master" for that cluster.  
All p4-managed processes (see the procedure `p4_create_procgroup') 
have unique integer id's beginning with 0.
The processes within a cluster are numbered consecutively.



Developing a Simple p4 Program
==============================


The real fun associated with any computing environment arrives when you
actually type in a program and run it yourself.  We will assume that
you have successfully installed p4 on your own system and are ready to
write a small program, compile it, and run it.



A Minimal Example
-----------------


We will start with a tiny program in which the slave processes do no work, and
then expand its capabilities.  Edit a file called `p4simple.c' and type:

     
         #include "p4.h"
     
         main(argc,argv)
         int argc;
         char **argv;
         {
             p4_initenv(&argc,argv);
             if (p4_get_my_id() == 0)
                 p4_create_procgroup();
             slave();
             p4_wait_for_end();
         }
     
         slave()
         {
             printf("Hello from %d++",p4_get_my_id());
         }
     


This is one of the simplest p4 programs that you can write.  Let's
examine it.  The `#include "p4.h"' statement must appear in all
programs that use any p4 features.  The procedure `p4_initenv'
must be invoked before any other p4 procedures, and
`p4_wait_for_end' must be invoked after all p4 processing is
completed.  The `p4_get_my_id' returns a unique integer id for
each process, beginning with 0.  The procedure
`p4_create_procgroup' should only be invoked once (by process 0),
and is responsible for creating all other processes.  The way in which
`p4_create_procgroup' determines how many other processes there
should be, and where they should run, will be discussed shortly.

All processes that this program executes invoke the slave procedure,
including process 0.  Thus, in this program, the master process acts
just like all other processes once it gets the environment
established.

To understand how things get started, let's consider two separate
situations.  In the first situation, all processes are running on a
single machine.  Then, when process 0 starts, it executes the
`p4_create_procgroup' procedure to start all other slaves.  The other
slaves are started on the same machine by means of a UNIX {fork},
and they immediately invoke a procedure named slave.  Thus, these local
slaves do not ever execute the main procdure.

In the second situation, there may be slaves running both on the same
machine as process 0, and slaves running on other machines as well.
In this situation, the first slave running on a remote machine will
need to execute the main procedure.  It will discover that it is not
process 0.  However, as part of initialization, process 0 will
direct it to fork any additional slaves required on the same machine.


In some ways, the above example can be used a a prototype for all p4 programs,
just by varying the content of the `slave' routine.  The routine name
`slave' is built into p4 in order to enable the same code to be run on
shared-memory machines via `fork' or on remote machines via `rsh'.
Different wrappers are automatically put around the `slave' code.  These
wrappers depend on the user code being called `slave'.  Thus your p4
program must contain a subroutine called `slave' in order to link
properly. 



A More Complicated Example
--------------------------


Now, let's make the slave process a little bit more interesting.  Let's
assume that we have `nprocs' slaves with ids 
0, 1, 2, ... `nprocs' -1.  And, we want to write a program in which
every process sends a single message to every other slave, and then
receives a message from every other slave.  We might alter the code for
the slave procedure to be the following:

         slave()
         {
             char *incoming, *msg = "hello";
             int myid, size, nprocs, from, i, type;
     
             myid = p4_get_my_id();
             nprocs = p4_num_total_ids();
             for (i=0; i < nprocs; i++)
             {
                 if (i != myid)
                     p4_send(100, i, msg, strlen(msg)+1);
             }
             for (i=0; i < nprocs - 1; i++)
             {
                 type = -1;
                 from = -1;
                 incoming = NULL;
                 p4_recv(&type,&from,&incoming,&size);
                 printf("%d received msg=:%s: from %d++",myid,incoming,from);
                 p4_msg_free(incoming);
             }
         }


This program demonstrates several features of p4's support for
message-passing.  Before we get into the specifics however, let's
examine the overall logic of the program.  Each process determines its
own id and the total number of processes executing in this run
(including process 0).  Then, in the first for-loop, each process sends
a single message to each of the other processes.  Finally, in the
second for-loop, each process receives a message from each of the other
processes.

The `p4_send' call requires 4 arguments:
   * a message type (arbitrarily chosen to be 100 here)
   * the id of the process to receive the message
   * the message itself
   * the size of the message


The use of `p4_recv' is slightly more complicated.  First, we
assign -1 to each of the parameters type and from.  This is done
because -1 represents a wildcard value indicating we are willing to
receive a message of any type from any process.  Here, we could have
coded type to be 100, and specified from equal to the value of i each
time through the loop (skipping our own id).  By setting incoming to
NULL, we have also indicated to `p4_recv' that we do not have a
buffer in which to place the received message, so `p4_recv' should
obtain a buffer for us and place the message in that buffer.
`p4_recv' treats these three parameters as both input and output
values.  Thus, it alters the value of each such that type and from
indicate the type of message received and the id of the process that
sent it.  The value of `incoming' is altered to point to the
buffer where the message was placed.  The `size parameter' is
strictly an output parameter and iindicates the size of the received
message.  It is possible for the user to provide his own buffer; this
will be demonstrated later.


Finally, note that `p4_msg_free' frees the message buffer obtained by
`p4_recv.' The procedure `p4_msg_free' should be called only after
the contents of the message are no longer needed.  `P4_msg_free' should
be used to free these buffers because, although a user only sees the data
portion of a message, p4 internally represents a message as a structured data
item.

To compile and link this program for execution, you need to create a
makefile.  We will assume that you have installed p4 in
`/usr/local/p4' and that you have typed the program above into a
file name `p4simple.c' in the directory
`/home/mylogin/p4pgms'.

To build your makefile, copy the file
         /usr/local/p4/messages/makefile.proto

into your working directory.  This is a prototype makefile that
contains machine-independent information, and which p4 can use to build a
machine-specific makefile for your program.  This prototype makefile contains
information about several sample programs that demonstrate
message-passing in p4.  If you edit this file, you will see information
for making a program named `sr_test'.  Do a global change of
`sr_test' to `p4simple'.  You should also change the value of
`P4_HOME_DIR'.  It should contain the full
pathname of the p4 system, e.g. `/usr/local/p4'.  Now change
directories to `/usr/local/p4' and type:

     make makefiles MACHINE=<machine_type> DIRS=/home/mylogin/p4pgms

where `<your_machine_type>' is the machine type that you specified when
you installed p4 on your machine.  Now, you should be able to change
back to your directory and see a file named Makefile there.  
You should then be able to type:

         make p4simple


There is one last piece missing before you can execute your program.
Recall that `p4_create_procgroup' needs to know how many processes
to start and where to start them; it reads a file (called a
"procgroup file") to gather this information.  p4 always assumes
that you have a master process, and that you describe the slave
processes (process groups) in the procgroup file.  You can name a
procgroup file any name you choose, but `procgroup' is the default
name.  The information contained in procgroup files can get fairly
involved, but if you have a computer that supports shared memory among
processes, then you can code a very simple example at first.

Let us suppose first that you want to run your program on a network of
workstations.  Then your procgroup should look something like:

         local O
         some.network.machine 1 /home/me/p4progs/p4simple


This file indicates that you wish to run only the master on the local machine
(the one you are logged into when you execute the program) and one slave on
the machine `some.network.machine'.

Now, all you have to do to run your program is type:

         p4simple


You should see a line printed each time a process receives a message
from another process (on some machines, there may be a restriction that
only one process can do I/O, however such restrictions are not
common).  Experiment by changing the number of slaves indicated in the
procgroup file.

You may notice that even a small p4 program becomes large when linked
with the p4 library.  You might consider using `strip' to reduce
the size or removing `-g' from the CFLAGS in the makefile.


Command-Line Arguments
======================



The command-line arguments to a p4 program are all optional.
     -help           get this information and the version number, then exit
     -pg             procgroup_file
     -dbg            set debug level 
     -rdbg           set remote debug level
     -gm             set globmemsize
     -dmn            provide local domainname
     -outfile        output file for master
     -routfile       output file prefix for remote masters
     -ssport         port# private port number for secure server
     -p4log          enable internal p4 logging by alog
     -p4version      print the current p4 version number




The p4 Function Library
=======================


Overview of the Library
-----------------------


In the following sections, we provide details for each p4 function in the
library.  The procedures are gathered into the following groups:

   * Functions for managing processes and clusters
   * Functions for message passing
   * Functions for shared memory
   * Functions for timing p4 programs
   * Functions for debugging p4 programs
   * Miscellaneous functions
   * Fortran interface functions



Return Codes from p4 Functions
------------------------------



Most p4 functions return -1 if an error occurs.  Some, however, call the
function `p4_error' when severe errors occur.  This function prints a
message and then attempts to terminate all of the user's processes
*Note Functions for Debugging p4 Programs::.


p4 Functions for Managing Processes and Clusters
================================================


In some situations a p4 procedure will give an error message and then
exit.  This is typically done as a result of a failed system call and
handled by calling the p4 procedure named `p4_error' that examines the
return values from socket procedures, etc.  Most of the time however,
the procedures simply return a value.  Some of the procedures return no
value and thus are declared to return `VOID'.  Some of the
procedures return either a pointer to a character string or `NULL';
`NULL' indicates an error.  The remaining procedures return an
integer value; (-1) indicates an error.



Functions for Process Management
--------------------------------



In this section we describe the p4 functions needed for basic creation and
termination of processes.


     int p4_initenv(argc,argv)
     int *argc;
     char **argv;

should be called by your program before an attempt is made to use
any p4 procedures or data areas.  We suggest making it the first executable
statement in your program.  `p4_initenv' parses the command line
arguments and extracts the ones intended for p4 ignoring all others (see the
discussion of command line arguments).  Note that you pass the address of
`argc' to `p4_initenv' so that it can actually remove its own
arguments before your program looks at them.


     int p4_create(fxn)
     int (*fxn)();
     
     
     int p4_create_procgroup()

There are two procedures that you can use to create processes in p4,
`p4_create_procgroup' and `p4_create'.  Processes created via
`p4_create' are said to be "user-managed" whereas those created
by `p4_create_procgroup' are "p4-managed".  The p4-managed
processes are automatically assigned unique id's (beginning with 0 for
the big master), they have message queues allocated for them so that
they can do message-passing, and they are able to run either on a
shared-memory multiprocessor with the creating process or they can run
on a separate machine.  Processes created via `p4_create' do not
have any of these advantages.  They must develop their own id's, they
cannot do message-passing, and they can only run on a shared-memory
multiprocessor with the creating process.  The only disadvantage of
`p4_create_procgroup' is that you must build a `procgroup'
file describing the set of required slave processes before the master
program begins execution.  This eliminates the possibility of
determining late in the execution exactly how many processes you want
to use to solve a problem.  Generally, this is not a problem,
especially since we can combine `p4_create_procgroup' and
`p4_create' in the following way: You can use
`p4_create_procgroup' to develop a network of processes that talk
to each other via messages.  Each of those processes can create further
processes to help it out as necessary.  The original set of processes
communicate with their local slaves through shared data areas and with
each other via message-passing.

`p4_create' receives one argument that is a pointer to a function.  It
creates a single new process that executes the indicated function.  The
new process may share data areas (in shared memory) with the parent
process.  However, the new process is not managed by the p4 system in
the sense that it is not assigned an id, it cannot pass messages, etc.
The only p4 procedure that deals with user-managed slaves is `p4_create'.
No other procedures are even aware of their existence.

`p4_create_procgroup' reads your `procgroup' file to determine the 
number of slave processes to create and where they are to be placed.  It
builds a procgroup table that describes all created processes and gives
a copy of the table to each process.  The processes then use the table
to discover how to communicate with each other (processes in a cluster
can send messages directly through shared memory or some other
vendor-specific mechanism), others communicate via sockets). 
An alternative method is to build the table in memory yourself and use
`p4_startup'.

The effect of `p4_create_procgroup' can be obtained in another way if a
system would prefer to use its own way of specifying the locations of
processes.  A user may allocate the procgroup data structure and then fill it
in "by hand" rather than by reading a file in p4 procgroup format.  The
following procedures support this method of starting processes.


     struct p4_procgroup *p4_alloc_procgroup()

allocates a procgroup data structure of the form described in `p4.h'.
The formats of individual entries (`p4_procgroup_entry') are given there
as well. 


     int 4_startup(pg)
     struct p4_procgroup *pg;

starts processes as specified by an an already-created procgroup data
structure allocated by `p4_alloc_procgroup' and filled in by the user
using the structures `p4_procgroup_entry' and `p4_procgroup'.



     VOID p4_wait_for_end()

is the p4 termination/cleanup procedure that you should invoke at the
end of every execution of a program that uses p4.  
It does some termination processing and then waits for slave processes 
to end.


     int p4_get_my_id()

returns an integer value representing the id of the process assigned by
the p4 system.  If the process is not a p4-managed process, the value
(-1) is returned.


     int p4_num_total_ids()

returns an integer value indicating the total number of ids started
by p4 in all clusters, including the big master and all remote masters.


     int p4_num_total_slaves()

returns an integer value indicating the total number of processes started
by p4 in all clusters, including all remote masters but not the big master.


Functions for Cluster Management
--------------------------------



The p4 system supports the "cluster" model of parallel computation, in
which subsets of processes share memory with one another, with the clusters
communicating via messages.  A procgroup file for a program written for the
cluster model might look like this:

         local 4
         alliant1.abc.edu     5 /home/me/myprog
         alliant2.abc.edu     5 /home/me/myprog
         encore.somewhere.edu 5 /usrs/me/myprog


This would specify a total of 20 processes, 5 (including the master) running
on the local machine (here assumed to be capable of supporting five processes
that share memory) together with 5 slaves each on three other shared-memory
machines.


     VOID p4_get_cluster_ids(start,end)     
     int *start;
     int *end;

receives pointers to two integers.  It places the p4-assigned id's of
the first and last ids within the current cluster into the two
arguments (including the remote master).


     int p4_get_my_cluster_id()

returns a unique id (relative to 0) within a cluster of p4-managed
processes.  Thus, a cluster master will always have a cluster id of 0.
It is not clear that a separate cluster id is really useful, but the
functionality is provided just in case.


     BOOL p4_am_i_cluster_master()

returns a BOOL value indicating whether the invoking process is the
"cluster master" process within its cluster.


     int p4_num_cluster_ids()

returns an integer value indicating the number of ids in the current
cluster as started by `p4_create_procgroup'.  



Functions for Message Passing
=============================



P4 supports a set of send/receive procedures.  These procedures are
"generic" in the sense that they do not know whether a message must
travel across a network or through shared memory, or via some other
mechanism.  They depend on a lower-level set of procedures that handle
local or network (remote) communications.  By default, the messages
are assumed to be typed.  If the user wishes to use untyped messages,
he can hide the typing by coding some very simple C macros that always
use a single message type.



Explicit Sending and Receiving of Messages
------------------------------------------











     p4_send(type,to,msg,len)
     p4_sendr(type,to,msg,len)
     p4_sendx(type,to,msg,len,datatype)
     p4_sendrx(type,to,msg,len,datatype)
     p4_sendb(type,to,msg,len)
     p4_sendbr(type,to,msg,len)
     p4_sendbx(type,to,msg,len,datatype)
     p4_sendbrx(type,to,msg,len,datatype)
     
     int type, to, len, datatype;
     char *msg;


Each of these procedures sends a message.  The `type' argument is
an integer value chosen by the user to represent a message type.  The
`to' argument is an integer value that specifies the p4-id of the
process that should receive the message.  The `len' argument
contains the length of the message to be passed.  Note that some of the
procedures have a "b" in their name, e.g.  `p4_sendb'.  These
procedures assume that the msg is in a buffer that the user obtained
earlier via a `p4_msg_alloc'; otherwise, the buffer is assumed to
be in the user's local space, and may cause the message to be copied
internally.  The procedures with an "r" in the name do not return
until an acknowledgement is received from the `to' process (the
"r" stands for rendezvous).  Those procedures with an "x" in the
name take an extra argument (datatype) that specifies the type of data
in the message; these procedures will use that information to call XDR
for data conversion if the message is being passed to a machine of a
different architecture, i.e. where the internal representation may be
different.



     BOOL p4_messages_available(req_type,req_from)
     int *req_type,*req_from;

returns a BOOL value indicating whether the process has any messages available
or not.  The parameters `req_type' and `req_from' are both pointers
to integers; they are used as {both} input and arguments.  On input,
`req_type' has a value that indicates the type of message that the user
wishes to check for availability (-1 indicates any type).  The variable
`req_from' is used similarly to indicate who a message is desired from.



     int p4_recv(req_type,req_from,msg,len_rcvd)
     int *req_type,*req_from,*len_rcvd;
     char **msg;

pointer to a `char'.  If this value is NULL, then p4 will allocate the
buffer for the message according to its length.  That is, one need not know
ahead of time the length of a message being received.  If this value is not
NULL, then it points to a p4 message buffer that the user has obtained via
`p4_msg_alloc'.  The `len_rcvd' argument is a pointer to an integer
that is assigned the length of the received message.  `Req_type' and
`req_from' are both pointers to integers; they are used as both input and
arguments.  On input, `req_type' has a value that indicates the type of
message that the user wishes to receive (-1 indicates any type).  It will
block until a message of that type is available.  `Req_from' is used
similarly to indicate who a message is desired from.  One important note about
this procedure is that it obtains the area in which to place a message, and
the user must explicitly free that area when finished with it (see
`p4_msg_free').  There is an option available with `p4_recv' in
which the user can provide his own buffer rather than having p4 allocate it.
To do this, the user points `msg' to a buffer that he must obtain via a
call to `p4_msg_alloc' (see below).  Then he assigns `len_rcvd' a
value which is the length of that buffer.  In this case, `len_rcvd' is
both an input and output variable.  In addition, no `p4_msg_free' need be
performed if the same buffer is going to be re-used multiple times.



     char *p4_msg_alloc(len)
     int len;

obtains a pointer to a buffer area that can be used to receive a 
message.  This procedure should be used for this task because a 
message has hidden information which the user is unaware of and 
therefore should not use malloc to obatin the area.



     VOID p4_msg_free(m)
     char *m;

frees the message pointed to by `m.'  This procedure should be used
for this task because a message has hidden information which the
user is unaware of and therefore cannot be freed by the user.



Global Operations
-----------------



P4 supports a number of operations for dealing with all processes at once.


     p4_broadcastx(type, data, data_len, data_type)
     int type;
     char *data;
     int data_len, data_type;



     p4_broadcast(type, data, data_len)
     int type;
     char *data;
     int data_len;

provide the ability to broadcast messages like `p4_send' and
`p4_sendx'.


     p4_global_op(type,x,nelem,size,op,data_type) 
     int type;
     char *x;
     int size, nelem;
     int (*op)();
     int data_type;

where `op' is one of:
     p4_int_absmax_op()
     p4_int_absmin_op()
     p4_int_max_op()
     p4_int_min_op()
     p4_int_mult_op()
     p4_int_sum_op()
     p4_dbl_absmax_op()
     p4_dbl_absmin_op()
     p4_dbl_max_op()
     p4_dbl_min_op()
     p4_dbl_mult_op()
     p4_dbl_sum_op()
     p4_flt_absmax_op()
     p4_flt_absmin_op()
     p4_flt_max_op()
     p4_flt_min_op()
     p4_flt_mult_op()
     p4_flt_sum_op()


and `data_type' is one of `P4INT', `P4LNG', `P4FLT', or
`P4DBL'.

This collection of routines provide the ability to do a variety of
global operations.  See the example programs in subdirectory
`p4/contrib'.  They apply the commutative operation `op' globally 
to `x' on an element-by-element basis and broadcast the result to 
all nodes.  That is each process ends up with

        for (i=0; i<n; i++)
              x[i] = x[node 0][i] op x[node 1][i] op x[node 2][i] op ...

`op' should be of the form

          VOID op(char *x, char *y, int nelem)
          {
              data_type *a = (data_type *) x;
              data_type *b = (data_type *) y;
     
              while (nelem--)
                  *a++ operation= *b++;
          }

where `data_type' and `operation' are chosen appropriately.

The order in which nodes apply the operation is undefined (hence
`op' must be commutative). The communication may be internally
sub-blocked so the function `op' should not be hardwired to specific
vector lengths.

This is still a relatively primitive version, which gathers the necessary data
up a balanced binary tree and then uses `p4_broadcast' to send the
results back. 



     VOID p4_global_barrier(type)
     int type;

This procedure takes one argument which is the message type to be 
used for internal message-passing.  It causes the invoking process to
hang until all processes specified in the procgroup file have invoked 
the procedure.



Functions for Shared Memory
===========================



Managing Shared and Local Memory
--------------------------------



     char *p4_malloc(n)
     int n;

typically acts like the standard `malloc', but may be rewritten for user 
systems that require different operation.


     VOID p4_free(p)
     char *p;

typically acts like the standard `free', but may be rewritten for
user systems that require different operation.


     char *p4_shmalloc()

acts like the standard `malloc' except will obtain shared memory on
machines that support sharing memory among processes.  Compare with
`p4_malloc'.


     VOID p4_shfree()

frees memory obtained with `p4_shmalloc'.  Compare with `p4_free'.



Shared Memory Data Types
------------------------



The abstraction provided by p4 for managing data in shared memory is
"monitors".  Good places to learn about the monitor concept in general are
[pbh:architecture] and [hoare:monitors].  The specific approach
taken by p4 is described in [lusk-overbeek:p4-book].  P4 provides several
useful monitors (`p4_barrier_t', `p4_getsub_monitor_t',
`p4_askfor_monitor_t') as well as a general monitor type to help the user
in constructing his own monitors (`p4_monitor_t').


Monitor-Building Primitives
---------------------------



The following functions can be used to construct monitors.  A monitor so
constructed has the type `p4_monitor_t'.


     int p4_moninit(m,i)
     p4_monitor_t *m;
     int i;

initializes the monitor pointed to by `m' and gives it `i' queues for
processes to wait on while they are blocked (see `delay').  One queue is
sufficient for most purposes.  The queues are numbered beginning with 0.


     VOID p4_menter(m)
     p4_monitor_t *m;

enter the monitor pointed to by m.  By the definition of a monitor,
access is restricted to a single process in the monitor at a time (if
everybody plays by the rules).


     VOID p4_mexit(m)
     p4_monitor_t *m;

exits the monitor pointed to by m.  You are of course assumed to have
previously entered that monitor.


     VOID p4_mcontinue(m,i)
     p4_monitor_t *m;
     int i;

checks to see if there are any processes blocked on the `i'-th queue of
the monitor `m' and causes one of them to be released for entry to the
monitor if so.  If there are no such processes, the invoking process
simply exits.  Note that a process could have been blocked previously
by invoking the procedure `p4_mdelay'.  The queues are numbered 
beginning with 0.


     VOID p4_mdelay(m,i)
     p4_monitor_t *m;
     int i;

permits a process to delay itself on the `i'-th queue of monitor
`m' if the process wishes to release the monitor, but wants to be
waked up by another process later (via the procedure `p4_mcontinue').
The queues are numbered beginning with 0.


Some Useful Monitors
--------------------



In this section we describe some of the specific monitors that are built into
the p4 library.  Each of them has its own pre-defined type, which can be
used to allocate storage for them, which should be in shared memory.
See the `p4/monitors' directory for examples.  A lock is itself a
monitor, with no extra delay queues.


     VOID p4_lock_init(l)
     p4_lock_t *l;

initializes the lock `l'.  Must be used prior to any attempts to lock or
unlock `l'.



     VOID p4_lock(l)
     p4_lock_t *l;

blocks if the lock `l' is already locked, otherwise locks `l'
and proceeds.



     VOID p4_unlock(l)
     p4_lock_t *l;

unlocks the lock `l'.


     VOID p4_getsub(gs,s,max,nprocs)
     p4_getsub_monitor_t *gs;
     int *s,max,nprocs;

is a procedure used to obtain the next value of a shared counter
(subscript).  It takes as its first argument, a pointer to a getsub
monitor that protects the shared counter.  It assigns the current value
of the counter to the integer that s points to, and then increments the
counter by 1.  `p4_getsub_init' initially sets the counter to 0.
When the counter passes the value `max', all `nprocs'
processes are returned the value (-1) once, then the counter is reset to
0 for further use.


     VOID p4_getsubs(gs,s,max,nprocs,stride)
     p4_getsub_monitor_t *gs;
     int *s,max,nprocs,stride;

is like `p4_getsub' except that the counter is increased on each call by
`stride' instead of 1.



     int p4_getsub_init(gs)
     p4_getsub_monitor_t *gs;

initializes the getsub monitor pointed to by `gs'; this initialization
includes assigning a value of 0 to the counter that the monitor
protects.


The standard barrier synchronization pattern is expressed as a monitor.
There can be multiple barrier monitors, and one can wait for only some
of the processes at the barrier if this is desired.



     VOID p4_barrier(b,nprocs)
     p4_barrier_monitor_t *b;
     int nprocs;

causes the executing process to hang until `nprocs' processes execute
a barrier instruction with a pointer to the same barrier monitor `b'
as an argument.


     int p4_barrier_init(b)
     p4_barrier_monitor_t *b;

initializes the barrier monitor `b'; this procedure should be invoked
before you attempt to use the monitor in any operations.

Finally, the `askfor' monitor functions like a general dispatcher of
work. 



     int p4_askfor(af,nprocs,getprob_fxn,problem,reset_fxn)
     p4_askfor_monitor_t *af;
     int nprocs;
     int (*getprob_fxn)();
     VOID *problem;
     int (*reset_fxn)();

requests a new "problem" to work on from the problem pool.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) the number of processes that call this procedure
(with `af') looking for work, (3) a pointer to the user-written procedure
that obtains a problem from the pool, (4) a pointer that is filled in
with the address of a user-defined representation of a problem to be
solved, and (5) a pointer to a user-written procedure to reset when all
problems in the pool are solved, in case the same monitor is re-used
for another set of problems later.  `p4_askfor' returns an integer
indicating whether a problem was successfully obtained or not:

         -1     : program is terminating (some process called p4_progend)
          0     : a problem was obtained and "problem" points to it
          1     : problem solved by exhaustion, i.e. no more problems to get
          n > 1 : a process found a solution and called p4_probend with code n

For a detailed discussion of the "askfor" monitor, see
[lusk-overbeek:p4-book].


     int p4_update(af,putprob_fxn,problem)
     p4_askfor_monitor_t *af;
     int (*putprob_fxn)();
     VOID *problem;

updates the problem pool being managed by the askfor monitor.  The
arguments are (1) a pointer to the askfor monitor that protects the
problem pool, (2) a pointer to the user-written procedure that puts
problems into the pool, and (3) a pointer to a user-defined
representation of a problem to be put in the pool.  `Putprob_fxn'
should return 1 if it did indeed put a new problem into the pool, so
that any delayed processes should wake up and re-examine the pool (this
logic is handled by the `p4_askfor') and 0 if upon entering the
monitor and examining its potential problem together with the data there
it decided not to add a new problem to the pool.  It can be assumed that
the "putprob" logic (defined by `putprob_fxn') is executed inside
the monitor.


     int p4_askfor_init(af)
     p4_askfor_monitor_t *af;

initializes the askfor monitor `af'; this procedure should be invoked
before you attempt to use the monitor in any operations.


     VOID p4_probend(af,code)
     p4_askfor_monitor_t *af;
     int code;

allows the user process to mark a problem as solved early when
several processes are coordinating their activities via an askfor
monitor.  The code is an integer value that will be returned to all
processes when they "askfor" a new sub-problem to work on.


     VOID p4_progend(af)
     p4_askfor_monitor_t *af;

allows a process to cause a return code of (-1) to be returned to all
processes using an askfor monitor.  This would typically be called by
a master process to indicate that no more problems are to be solved
and that all slave processes should terminate.


Functions for Timing p4 Programs
================================



A small number of simple functions are available for accessing various
clocks and timers.  


     int p4_clock()

returns a value in milliseconds.  This is a wall-clock value, usually obtained
from the system via `gettimeofday'.  Also see `p4_ustimer' below.


     p4_usc_time_t p4_ustimer()

returns a wall-clock time value in microseconds.  The precision of this
number depends on the timer installed on the individual machine.  In
some cases the resolution may be no greater than that of `p4_clock()'.
For arithmetic and printing purposes, the type `p4_usc_time_t' is an
unsigned long integer.


     p4_usc_time_t p4_usrollover()

returns the timer value at which a microsecond timer "rolls over".
Since `p4_usc_time_t' is a long integer's worth of microseconds, it is
likely that the timer will roll over (become zero) during even
medium-length runs.


Functions for Debugging p4 Programs
===================================



P4 has a set of routines to aid in producing a printed trace of events, both
user-defined and pre-defined in the p4 system.


     VOID p4_dprintf(fmt, va_alist)
     char *fmt;
     va_dcl

acts just like the standard `printf' except that the print line is
preceded by a value that identifies the process.  This value is
typically the string `pn_u' where `n' represents the
p4-assigned id and `u' represents the unix-id of the process on its
host.  However, there are other forms of this value.  For example, the
big master is represented as `bm_u'.  Also, if a process prints
before it has a p4-assigned id, then its value will be something like
`bm_slave_n_u' or `rm_slave_n_u'.  Typically, it is not
possible for a user program to print anything before being assigned an
id by p4, but the p4 system itself may use this procedure to print
messages from a particular process if it encounters problems getting the
process initialized.


     VOID p4_dprintfl(level, fmt, va_alist)
     int level;
     char *fmt;
     va_dcl

is like `p4_dprintf' except that the first argument is an integer
indicating the debugging level that must be in effect before this
message will print.  A level of 0 will cause the message to always print.
If you run a program with the debug level set to 5 (via command-line
arguments), then all `dprintfl''s with level less than or equal to
that debug level will print. *Note Command-Line Arguments:: for how to
set the debug level at run time.

The debug level can be examined and changed by the user during execution:


     int p4_get_dbg_level()     

returns the current debug level for this process and its cluster.


     VOID p4_set_dbg_level(level)     
     int level;

sets the current debug level for this process and its cluster.
P4 itself is liberally instrumented with `p4_dprintfl''s of level
10 and above, leaving levels 0-9 for the user.  The greater the debug
level of the built-in messages, the greater understanding of p4 needed
by the user to make sense of them.  However, levels as high as 30 may
well be useful to the user trying to debug a p4 program.
Roughly speaking, the following debug levels produce messages about the
indicated events.

     level 10:  created process
                sent message
                received message
     
     level 20:  creating process
                sending message
                receiving message
                process starting
                process exiting
     
     level 30:  waiting for ack
                sending ack
                sent ack
                received ack
                queueing message for later receipt
                queued message for later receipt
     
     level 40:  memory management
                buffer management
     
     level 50:  reading procgroup
                other initialization message exchange
     
     level 60:  send-receive details, especially machine-specific traces
     
     level 70:  listener interactions:
                 creating listener
                 created listener
                 messages from inside listener
     
     level 80:  detailed data structures after initialization
     
     level 90:  detailed tracing of flow thru procedures


For optimum performance, the test of the debug level required by these
messages can be removed at compile time by not commenting out the 
`#define P4_DPRINTFL' line in the `OPTIONS' file.
(*Note Structure of the Distribution Directory::).

The following function is provided to deal with abnormal termination.
It can be called by any process.


     VOID p4_error(string, value)
     char *string;
     int value;

prints `string' as an error message and then forcefully terminates
all co-operating processes and cleans up all shared resources.


     VOID p4_soft_errors(onoff)     
     int onoff;

enables/disables soft errors, returning the previous setting.  The default
is "disabled", which means that certain p4 functions will call
`p4_error' instead of returning -1.


`p4_error' gets control on certain kinds of interrupts.  It is
automatically called for `SIGSEGV', `SIGBUS', and `SIGFPE'
interrupts, to catch user programming errors and clean up, after which it
returns interrupt handling to default mode and returns, so that the user
may obtain a dump.  It also handles `SIGINT' interrupts, in which case
it cleans up and exits.  Finally, it may be called directly by the user, in
which case it cleans up (other p4 processes and IPC's) and exits.


Although `p4_error' is supposed to get rid of all running p4
processes, it can happen that an error is bad enough that p4 processes
are left running.  A primitive aid in finding and killing these
processes is the shell script `kj', which takes a string as an
argument and then kills processes containing that string as part of
their program names.  Currently it only kills processes on the machine
where it is run, but it can be run via `rsh' on remote machines.
There are other useful scripts (e.g. `killipc' and `killp4')
in the `p4/bin' directory to do such things as clean up SYSV IPC 
items that may be left when a program abnormally terminates.  P4 will 
generally cleanup these items if the abnormal termination is a type that 
p4 traps, otherwise the user must do the cleanup.  This is an unfortunate
side-effect of the way that SYSV handles things, it really should be
the OS's function to take care of this.

On many machines it is possible to attach a debugger like `dbx' to a
running process.  This is one way to find out where a hanging process is
stuck. 

Miscellaneous Functions
=======================


In this section are found functions that do seem to fit neatly into any of the
other sections.


     char *p4_version()

returns a string containing the version number of p4 being run.



     VOID p4_get_cluster_masters(numids, ids)
     int *numids, ids[];

This procedure fills in the values of numids and ids.  It obtains the
p4-ids of all "cluster masters" for the program, placing them in the
ids array and placing the number of ids in numids.


     VOID p4_print_avail_buffs()

P4 maintains an array of buffer lists of various sizes, so that it can very
rapidly allocate and deallocate buffers.  You can see the contents of the
buffer pools at any time by calling this procedure.



     VOID p4_set_avail_buff(bufidx,size)
     int bufidx;
     int size;

This procedure is used to set the size of buffers in p4's buffer pools.  The
parameter `bufidx' specifies a particular buffer list, and should be a
number from 0 to 7.  The `size' parameter specifies that buffers up to
that size will be managed by p4 in a particular list.  It is important to
maintain the buffer sizes in increasing order.  The default list of buffer
sizes is {64, 256, 1024, 4096, 16384, 65536, 262144, 1048576}.  This causes
wasted space if you send only one large message, causing the allocation of a
large buffer which is not reused.  Savings in space can be achieved by
adjusting these numbers to correspond with the message sizes of your
application.  If no large messages are sent at all, however, no space is
wasted since the large buffers will never be allocated.  If you send a message
larger that the largest size in this array, p4 will allocate the buffer, and
then free it back to the system as soon as it can.



Fortran Interface
=================



The Fortran calls to p4 procedures are analogous to their C counterparts, but
have Fortran-like names.

     #include "p4f.h"
     
     p4init()
     p4crpg()
     p4cleanup()
     p4sendr(type,dest,msg,len,rc)
     p4sendrx(type,dest,msg,len,data_type,rc)
     p4send(type,dest,msg,len,rc)
     p4sendx(type,dest,msg,len,data_type,rc)
     p4recv(type,from,buf,buflen,msglen,rc)
     p4brdcst(type,data,len,rc)
     p4brdcstx(type,data,len,data_type,rc)
     p4probe(type,from,rc)
     p4myclid()
     p4nclids()
     p4getclids(start,end)
     p4myid()
     p4clock()
     p4ustimer()
     p4ntotids()
     p4error(str,val)
     p4softerrs(new,old)
     p4version()
     p4flush()
     p4globop(type,x,nelem,size,op,data_type,rc)
        p4intsumop()
        p4intabsmaxop()
        p4intabsminop()
        p4intmaxop()
        p4intminop()
        p4intmultop()
        p4dblsumop()
        p4dblabsmaxop()
        p4dblabsminop()
        p4dblmaxop()
        p4dblminop()
        p4dblmultop()
        p4fltsumop()
        p4fltabsmaxop()
        p4fltabsminop()
        p4fltmaxop()
        p4fltminop()
        p4fltmultop()


The `data_type' parameter in the above operations shoudl be one of
`P4INT', `P4LNG', `P4FLT', or `P4DBL'.
These symbolic constants are defined in the include file `p4f.h'.


Faster Startup with the Secure Server
=====================================






P4 processes on remote machines are ordinarily created by `rsh'.
For this to work, the user must have permission to create processes on
that machine.  This permission is normally granted either globally by
the system administrator, or locally by the use of `.rhosts' files.
(See the normal unix man pages under `rhosts').

Since `rsh' is relatively slow, p4 provides a way to get things
started faster.  This is accomplished by running the program
`serv_p4' in the background on the remote machine.  When p4 is
creating processes, it will automatically check for the existence of
this server and use it if it is running.  Remote processes typically
start much faster when the server is running.  A disadvantage of using
the server is that output from `p4_dprintf' is lost unless directed
by a command-line argument to a file.  (*Note Command-Line Arguments::.)
When p4 uses `rsh', the remote process's `stdout' is sent
back to the `stdout' of the parent (the p4 master process).
We have not yet tested this server on all of the machines that we support.
Thus far, we have tested it somewhat on the SYMMETRY, SUN, DEC500, and
SGI.  We believe that it will work on many other machines, but have not
yet verified it on all machines.

An invocation of a set of servers is (currently) associated with a specific
port number.  This way multiple users can each be running multiple server
networks without mutual interference, provided each network of servers is
started with a different port number. 

To start the secrue server on a machine one can do

     serv_p4 -d -p <num>


where `<num>' is a port number to be associated with a network of
servers.  If the `-p' option is omitted, the server will pick an unused
port number and report

     Listening on <num>.


Then p4 programs to use this network should be started with 

     -ssport <num>


The p4 application must also be listed in the user's `.p4apps' file in
his home directory.  This file should be readable only by the user, and should
contain the full path names of programs that the user wishes to be startable
by the p4 server.

When a p4 master process tries to start a slave process on a remote machine,
it will first attempt to do it via the server.  If it cannot do so for any
reason (no server running, port number mismatch, or program not found in
`.p4apps' file), then it tries to do so with the remote shell command.

Note that the server is used only to start processes; it plays no role in a p4
computation once the slave processes have been initiated.  Rather, a temporary
process, called the {listener}, is spawned to manage connection requests
that occur during the execution of a p4 program.  Neither the server nor the
listener consumes any significant amount of CPU time.

There is further discussion of installation options for the servers in
the `README' file in the `p4/servers' subdirectory.


Utilities for Managing a p4 Session
===================================




A number of useful utilities can be found in the `bin' subdirectory.
These can be used to start and stop server processes based on the contents of
a file of machines one regularly uses, to kill runaway p4 processes in the
unlikely case that they cannot or do not terminate automatically when one
processes ends abnormally or interrupted from the keyboard, and to merge
logfiles created for the use of `upshot' *Note Creating Logfiles for Upshot::. 


Creating Logfiles for Upshot
============================






P4 is distributed with a set of routines for creating logfiles (see
`README' in the `p4/alog' directory.  The resulting logfiles
can be examined by `upshot', distributed separately.  For details
about `upshot', see [herrarte-lusk:upshot].  

The `p4/alog' directory contains a package (ALOG) for creating logs of
time-stamped events, that is of general utility, outside of p4.  The
timestamps are obtained from various microsecond-level resolution timers on
various machines.  The portable microsecond timing package is contained in the
`usc' subdirectory.  It is used by the ALOG package as well as by the
`p4_ustimer' function in p4.  Similarly, the ALOG package can be used
independently of p4 and `upshot'. Its logfiles were designed to be read
and displayed by upshot, but other display packages can be used as well.



User-Specified Events
---------------------



The ALOG package consists of a set of macros that can be used to instrument a
C program and a set of functions that can be used to instrument a Fortran
program.  We will focus here primarily on the use of the C interface, which
contains more functionality.

The macros that can be used to instrument a program are as follows (from the 
file `README_ALOG' in the `alog' directory):


     ALOG_SETUP(pid,flag):	
       pid  -  (integer) process id of callee
       flag -  (integer) either ALOG_WRAP or ALOG_TRUNCATE  

This macro initializes the tracing area for a slave process and must be called
once before any event is logged.  If the value of `flag' is set to
`ALOG_WRAP', then in the event of no more space for logging events the
system will only report the latest n events.  If `flag' is set to
`ALOG_TRUNCATE' the system will stop logging events as soon as there is
no more memory for the events to be logged.


     ALOG_MASTER(pid,flag):
       pid  -  (integer) process id of the callee
       flag -  (integer) either 0 or 1  (see above)

`ALOG_SETUP' with the difference that this macro should be referenced by
the master process only.


     ALOG_DEFINE(event,strdef,format):
       event  - (integer) id of event being defined
       strdef - (string) description of 'event'
       format - (string) control string in "printf" format to

This macro puts an event definition code into the logfile.


     ALOG_LOG(pid,event,intdata,strdata):
       pid     - (integer) process id of callee
       event   - (integer) event id to be logged
       intdata - (integer) any integer data for this event
       strdata - (string) any string data (can be the null string)

This macro provides the event logging service.


     ALOG_OUTPUT
       no parameters  

This macro dumps the events logged into a log file with the name
`alogfile.pxx' where `xx' is the logical PID of the callee process.
The log file is created in the current directory unless specified otherwise
through the macro ALOG_SETDIR.


     ALOG_SETDIR(dir)
       dir - (string) directory where log file is created

This macro sets the output directory for the log file.  The default directory
for the creation of the log file is the current directory of the process.  If
used, then this macro MUST be invoked before `ALOG_MASTER/ALOG_SETUP'.


     ALOG_STATUS(status):
       status - (integer) either ALOG_ON or ALOG_OFF


This macro controls the logging status of `ALOG' as follows.  Setting
`status' to `ALOG_ON' enables logging until it is turned off.
Setting `status' to `ALOG_OFF' disables logging until it is turned
on again.  Logging is enabled at the outset by default.



     ALOG_ENABLE
       no parameters


This macro enables event logging; same as calling `ALOG_STATUS(ALOG_ON)'.


     ALOG_DISABLE
       no parameters

This macro disables event logging; same as calling
`ALOG_STATUS(ALOG_OFF)'.

The sample program `gridlog.shmem.c' in the `monitors' subdirectory
contains an example of a program instrumented with ALOG statements.  The macro
definitions for ALOG are included when you include `#include "p4"' in
your program.  If the line `#define ALOG_TRACE' is not included before
the `#include "p4"', these macros will generate no code.  Thus it is
easy to effectively de-instrument the code by recompiling, and there is no
need to protect each ALOG statement with an `#ifdef'.
 
When an ALOG-instrumented program is run, it will produce one logfile for each
process.  The files will be named `alogfile.p0', `alogfile.p1',
.  These files need to be merged into a single file with the events
sored by timestamp.  This is accomplished with the program `mergelogs',
found in the `bin' subdirectory.  To merge the logfiles, do 

     mergelogs alogfile.p* > myprog.log
     rm alogfile.p*


The resulting logfile can be examined by upshot or some other logfile
examination facility.  See [herrarte-lusk:upshot] for details of the
logfile format.

On networks of workstations and some distributed memory machisnes, the
microsecond timers on the various processors are synchronized.  To produce a
usable merged logfile, the `adjlogs' program, also found in the
`bin' directory, can be used to adjust the timestamps for offset and
drift before they are merged.  For this to work, synchronization events must
be placed in the logfiles by an `ALOG_LOG' statement.  The event type
is then passed to `adjlogs', which aligns the timestamps, based on the
timestamps of the synchonization events.  The call to `adjlogs' looks
like this, where `<n>' is the type of the synchronization event.

     adjlogs -e <n>


Both `mergelogs' and `adjlogs' are less portable than the other p4
code;  you might want to run them on a workstation such as a Sun.


Examining Log Files with Upshot
-------------------------------




`Upshot' is not part of the p4 distribution, but can be obtained from the
same anonymous `ftp' location as p4.  Take the file `upshot.tar.Z'
from the directory `pub/p4' on `info.mcs.anl.gov'.  The distribution
contains all necessary documentation on how to install and run `upshot'.
It is an X-window program that runs on most workstations.  There is no need
for a parallel macchine to be involved, once the log files have been obtained.

`Upshot' produces the most interesting displays when certain events
(not necessarily all) are defined to be the entry and exit events for certain
{states} and then colors are associated with the states.  This association
is reflected in a {statefile} with a format like the following:

     1 1 2 red   asking
     2 3 4 blue  working
     3 5 6 green updating


This statefile describes three states.  State 1 is defined to be between
events 1 and 2.  `Upshot' will color it red and label it "asking".


Automatic Logging of p4 Events
------------------------------



We have found that the most useful events to log and study are those
identified by the user and specified in his program.  That way he can control
the number of events to be logged and the grain size of the states that are
represented.  

In some cases, however, one wants to study the details of the internal
operation of a p4 application, or get some idea of the behavior on one's
program without going to the trouble of instrumenting it himself.  To this
end, p4 itself is instrumented with ALOG statements, although by default they
are inactive.  To get automatic logging of p4 events (including sending and
receiving of each message) one needs first to link to a version of the p4
library that has been compiled with the line `#define ALOG_TRACE'
uncommented out in the `OPTIONS' file, and secondly, to run with
`-p4log' on the command line.



Machine-Specific Notes
======================


     SUN
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     HP
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not tested (not avail on our test machine).
     
     DEC5000
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     RS6000
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     IBM3090
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not supported due to absence of iargc/getarg.
         (3)  There are multiply defined macros in include/rpc/rpc.h.  IBM
              is fixing this in a later OS release.  Meanwhile make your own
              copy and the fix the problem yourself.
         
     TITAN
         (1)  P4 can be installed on this machine with or without SYSV IPC.
         (2)  Fortran not supported due to problems with getting args.
     
     SGI
         (1)  P4 can be installed on this machine with or without SYSV IPC.
     
     NEXT
         (1)  Fortran not supported due to absence of iargc/getarg.
     
     FX2800/FX2800_SWITCH
         (1)  Alliant's switch code not yet ensuring messages 
              remain ordered.  p4 currently discovers the switch port for
              the machine it is running on by invoking the internal
              procedure getswport.  This procedure must be customized to
              the installation.
           
     FX8
         (1)  You might need to add MFLAGS = -i to the Makefile
     
     KSR
         (1)  The latest version of the OS produces a link-time error for
              Fortran programs.
     
     IPSC860
         (1)  the script "runcube" (in the messages directory) may 
              be useful
     	 
     DELTA
         (1)  the script "rundelta" (in the messages directory) may 
              be useful
     
     BALANCE
         (1)  Fortran not supported.
     
     SYMMETRY/SYMMETRY_PTX
         (1)  -Z compiler option may be used to control shmalloc/malloc split
         (2)  shared memory message passing not supported in Fortran
     
     TC_2000/TC_2000_TCMP
         (1)  TCMP port not yet complete.
         (2)  For shared-memory execution, one must use `cluster ...' to
              obtain a private cluster for execution




Some Common Problems and their Solutions
========================================




Our attempt with this manual has been to prevent you from having difficulties.
Experience shows that certain common progblems recur, however.  In this
section we hope to address some of these problems.

"Permission Denied."     
      p4 slave processes are started by forks (for
       slaves in the same shared-memory cluster), by the server, or by the remote
       shell command.  If the server is running on the target machine then that
       must be configured to allow remote processes to be started.  To test whether
       this is your problem, try
              rsh target.machine date
     
       If you still get the "Permission denied." message, then the problem has
       nothing to do with p4.  See `hosts.equiv' or `.rhosts' in the
       system man pages.
"More processes than message queues"     
       Under the default configuration
       of p4, uniprocessors, such as most workstations, cannot have multiple
       process sharing memory.  Thus your procgroup file for a workstation network
       should always look like
              local 0
              machine1 1 pathname
              machine2 1 pathname
              machine3 1 pathname
                 .
                 .
     
       The "local" means "only the master on the startup machine; no local
       slaves sharing memory".
     
     
       It is possible, at some cost in message-passing efficiency, to have a
       cluster of processes sharing memory on a workstation, but in this case p4
       must have been installed with the `SYSV_IPC' option set in the
       `OPTIONS' file.  The cost is that a process waiting for a message must
       spin between checking for a message arriving on a socket and a message
       arriving through shared memory. 
     
"p4error: local is not first entry in procgroup"     
       the first line of
       the procgroup file must be the "local" entry, specifying the number of
       slaves that will be run on the master machine in addition to the master
       process. 
     
"gethostbyname failed 100 times"     
      Check for an invalid machine name in
       the procgroup file.
"pgmpathname: Command not found"     
       P4 tried to start the program
       with the given name on a remote machine and the program did not exist.
       Verify the full path name of the program.
program hangs     
       You may have failed to initialize the `type' and
       `from' fields before a `p4_recv'.  You might have used
       `p4_sendr' between two processes at the same time, which will deadlock
       if you think about it, or even if you don't.  Use `p4_send' instead.
program hangs or has bad data in received message     
       You might have
       failed to set the pointer to the incoming buffer to NULL, or to have
       specifically allocated a buffer with `p4_msg_alloc', before a
       `p4_recv'.
program ignores command-line arguments     
       You might have passed
       `argc' instead of `&argc' to `p4_initenv'.
program runs out of memory     
       You may need to call `p4_msg_free'
       after each `p4_recv', or reuse buffers by pre-allocating them.
     
     

SHAR_EOF
if test -f 'p4.bib'
then
	echo shar: over-writing existing file "'p4.bib'"
fi
cat << \SHAR_EOF > 'p4.bib'
@Book{lusk-overbeek:p4-book,
  author = 	"J. Boyle and R. Butler and T. Disz and B. Glickfeld 
                 and E. Lusk and R. Overbeek and J. Patterson and R. Stevens",
  title = 	"Portable Programs for Parallel Processors",
  publisher = 	"Holt, Rinehart, and Winston",
  year = 	1987
}

@Article{hoare:monitors,
  author = 	"C. A. R. Hoare",
  title = 	"Monitors:  an operating system structuring concept",
  journal = 	"Comunications of the ACM",
  year = 	1974,
  pages = 	"549-557",
  month = 	"October"
}

@Book{pbh:architecture,
  author = 	"Per Brinch Hansen",
  title = 	"The Architecture of Concurrent Programs",
  publisher = 	"Prentice-Hall, Inc.",
  year = 	"1977",
  OPTaddress = 	"Englewood Cliffs, New Jersey",
}

@TechReport{herrarte-lusk:upshot,
  author = 	"Virginia Herrarte and Ewing Lusk",
  title = 	"Studying Parallel Program Behavior with {\tt Upshot}",
  institution = "Argonne National Laboratory",
  year = 	1991,
  number = 	"ANL--91/15",
  address = 	"Argonne, IL 60439"
}

SHAR_EOF
cd ..
if test ! -d 'include'
then
	mkdir 'include'
fi
cd 'include'
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib
LIBS = $(MDEP_LIBS)
LIBLINK = ../bin

default:	

all:	

realclean:	clean
	$(RM) -f *.h Makefile

SHAR_EOF
cd ..
if test ! -d 'lib'
then
	mkdir 'lib'
fi
cd 'lib'
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

P4_HDRS = p4.h p4_sys.h p4_defs.h p4_macros.h  \
          p4_sock_util.h p4_MD.h p4_funcs.h p4_sys_funcs.h \
	  p4_mon.h p4_sr.h p4_globals.h p4_config.h

P4_COMM_OBJ	= 	p4_globals.o p4_MD.o p4_utils.o \
			p4_args.o p4_alloc.o p4_debug.o \
			p4_procgroup.o p4_tsr.o \
			p4_mon.o p4_broadcast.o p4_error.o p4_bm.o 

P4_SOCK_OBJ	=	p4_rm.o      p4_sock_util.o \
			p4_sock_cr.o p4_sock_conn.o \
			p4_sock_sr.o p4_sock_list.o \
			p4_secure.o

P4_CUBENODE_OBJ	=	$(P4_COMM_OBJ) p4_node_rm.o

P4_CUBEHOST_OBJ	=	$(P4_COMM_OBJ) $(P4_SOCK_OBJ)

P4_SHMEM_OBJ	=	p4_shmem_sr.o

LIBS = $(LIBDIR)/libp4.a $(MDEP_LIBS) 

$(P4_OBJ): $(P4_HDRS)

p4_cmain.o: $(P4_HDRS)

default:$(P) 
	make $(MFLAGS) libp4.a p4_cmain.o  

all:$(P) 
	make $(MFLAGS) libp4.a p4_cmain.o  

libp4.a:$(P) $(P4_OBJ)
	$(AR) $@ $?
	$(RANLIB) $@

realclean: clean
	$(RM) -f Makefile libp4.a alog*.h p4_config.h alog*.h usc*.h

#p4_funcs.h:
#	-mv p4_funcs.h p4_funcs.h.bak
#	csh make_funcdecls p4*.c > p4_funcs.h

#lintlib llib-lp4.ln:	lint_p4.c
#	@if [ $(UNIX) != P4BSD ] ;\
#	then\
#		echo lint $(LINTFLAGS) -x -op4 lint_p4.c ;\
#		lint $(LINTFLAGS) -x -op4 lint_p4.c ;\
#	else\
#		echo lint $(LINTFLAGS) -Cp4 lint_p4.c ;\
#		lint $(LINTFLAGS) -Cp4 lint_p4.c ;\
#	fi
SHAR_EOF
if test -f 'p4.h'
then
	echo shar: over-writing existing file "'p4.h'"
fi
cat << \SHAR_EOF > 'p4.h'
#include <ctype.h>
#include <stdio.h>

#if defined(RS6000)  ||  defined(SYMMETRY_PTX)
#include <sys/select.h>
#endif

/* for xdr  -  includes netinet/in.h and sys/types.h */
#include <rpc/rpc.h>      

#include <signal.h>
#include <errno.h>
#include <sys/time.h>
#include <pwd.h>
#include <fcntl.h>

#include "p4_config.h"
#include "p4_MD.h"
#include "p4_mon.h"
#include "p4_sr.h"

#include "p4_funcs.h"

#ifndef P4_DPRINTFL
#define p4_dprintfl
#endif

#include "alog.h"

#include "usc.h"
#define p4_ustimer() usc_clock()
#define p4_usrollover() usc_MD_rollover_val

#define HOSTNAME_LEN 64

struct p4_procgroup_entry {
    char host_name[HOSTNAME_LEN];
    int numslaves_in_group;
    char slave_full_pathname[100];
    char username[10];
};

#define P4_MAX_PROCGROUP_ENTRIES 32
struct p4_procgroup {
    struct p4_procgroup_entry entries[P4_MAX_PROCGROUP_ENTRIES];
    int num_entries;
};


SHAR_EOF
if test -f 'p4_MD.c'
then
	echo shar: over-writing existing file "'p4_MD.c'"
fi
cat << \SHAR_EOF > 'p4_MD.c'
#include "p4.h"
#include "p4_sys.h"

/* --------- Most Machine Dependent Stuff is in this file */

#if defined(ALLIANT) && defined(USE_XX_SHMALLOC)
char *xx_shmalloc();
P4VOID xx_shfree();
P4VOID xx_init_shmalloc();
#endif

#if defined(KSR) && defined(USE_XX_SHMALLOC)
char *xx_shmalloc();
P4VOID xx_shfree();
P4VOID xx_init_shmalloc();
#endif

P4VOID MD_initmem(memsize)
int memsize;
{
#ifdef TC_2000
    MD_malloc_hint(HEAP_INTERLEAVED | HEAP_UNCACHED, 0);
#endif

#if defined(GP_1000)
    xx_malloc(0, memsize);
#endif

#if defined(KSR) && defined(USE_XX_SHMALLOC)
#define  UNMAPPED  (caddr_t) -1
    caddr_t memory;
    unsigned size = ((memsize + 4095) / 4096) * 4096;

    /*   create shared memory  */
    memory = (char *) mmap( NULL, memsize, PROT_READ | PROT_WRITE,
                               MAP_ANONYMOUS | MAP_VARIABLE |MAP_SHARED,
                               -1,0);
    if ( memory == UNMAPPED)
    {
        p4_error("OOPS: mmap failed\n",memory);
    }
    xx_init_shmalloc(memory, size);
#endif

#if defined(ALLIANT) && !defined(USE_XX_SHMALLOC)
    xx_malloc(0, memsize);
#endif

#if defined(ALLIANT) && defined(USE_XX_SHMALLOC)
    unsigned size = ((memsize + 4095) / 4096) * 4096;
    char *memory = valloc(size);
    long id;
    if (!memory)
	p4_error("MD_initmem: failed in valloc", size);

    id = (long) getpid();	/* get a unique id */
    if (create_shared_region(id, (char *) memory, size, 0))
	p4_error("MD_init_mem: failed in create_shared_region", size);
    xx_init_shmalloc(memory, size);
#endif

#if defined(SYSV_IPC) && defined(USE_XX_SHMALLOC)
    int i,nsegs;
    unsigned size, segsize = P4_SYSV_SHM_SEGSIZE;
    char *mem, *tmem, *pmem;

    if (memsize  &&  (memsize % P4_SYSV_SHM_SEGSIZE) == 0)
	nsegs = memsize / segsize;
    else
	nsegs = memsize / segsize + 1;
    size = nsegs * segsize;
    if ((sysv_shmid[0] = shmget(getpid(),segsize,IPC_CREAT|0600)) == -1)
    {
	p4_error("OOPS: shmget failed\n",sysv_shmid[0]);
    }
    if ((mem = (char *)shmat(sysv_shmid[0],NULL,0)) == (char *)-1)
    {
	p4_error("OOPS: shmat failed\n",mem);
    }
    sysv_num_shmids++;
    nsegs--;
    pmem = mem;
    for (i=1; i <= nsegs; i++)
    {
	if ((sysv_shmid[i] = shmget(i+getpid(),segsize,IPC_CREAT|0600)) == -1)
	{
	    p4_error("OOPS: shmget failed\n",sysv_shmid[i]);
	}
        if ((tmem = (char *)shmat(sysv_shmid[i],pmem+segsize,0)) == (char *)-1)
        {
            if ((tmem = (char *)shmat(sysv_shmid[i],pmem-segsize,0)) == (char *)-1)
            {
                p4_error("OOPS: shmat failed\n",tmem);
            }
	    else
	    {
		mem = tmem;
	    }
        }
	sysv_num_shmids++;
	pmem = tmem;
    }
    xx_init_shmalloc(mem,size);
#endif

#if defined(MULTIMAX)
    share_malloc_init(memsize);
#endif
}

P4VOID MD_initenv()
{

#ifdef P4BSD
    struct timeval tp;
    struct timezone tzp;

    gettimeofday(&tp, &tzp);
    p4_global->reference_time = tp.tv_sec;
#endif
#ifdef P4SYSV
#   if defined(IPSC860)
    p4_global->reference_time = mclock();
#   else
    time_t t;

    time(&t);
    p4_global->reference_time = t;
#   endif
#endif

#if defined(FX2800_SWITCH)
    sw_attach(p4_global->application_id);
#endif
}				/* MD_initenv */

/* MD_malloc_hint variables.  Needs to be initialized appropriately
   for each system. */

#if defined(TC_2000)

static int characteristic = HEAP_INTERLEAVED | HEAP_UNCACHED,	/* "flag" arg to
								 * heapmalloc(3) */
  locality = HEAP_ANYWHERE;	/* "node" arg to heapmalloc(3) */
#endif

P4VOID MD_malloc_hint(a, b)
int a, b;
{
#if defined(TC_2000)
    characteristic = a;
    locality = b;
    if (a == -1)
	xx_malloc(2, b);	/* setting mapped filename */
    else if (a == -2)
	xx_malloc(3, 0);	/* doing heapsync() */
#endif
}

char *MD_shmalloc(size)
int size;
{
    char *p;

#if defined(BALANCE) || defined(SYMMETRY) || defined(SYMMETRY_PTX)
    p = shmalloc(size);
#else

#if defined(GP_1000) || defined(TC_2000)
/*   size = (size | 0xff) + 1;	 $$$ Why?  Nuked for now...  -jb */
    p = xx_malloc(1, size);
#  ifdef MALLOC_STATS
      allocated += size;
#  endif
#else

#if defined(KSR) && defined(USE_XX_SHMALLOC)
    p = xx_shmalloc((unsigned) size);
#else

#if defined(ALLIANT) && defined(USE_XX_SHMALLOC)
    p = xx_shmalloc((unsigned) size);
#else
#if defined(ALLIANT) && !defined(USE_XX_SHMALLOC)
    p = xx_malloc(1, size);
#else

#if defined(MULTIMAX)
    p = share_malloc(size);
#else

#if defined(SYSV_IPC) && defined(USE_XX_SHMALLOC)
    p = xx_shmalloc((unsigned) size);
#else

    p = (char *) p4_malloc(size);
#endif
#endif
#endif
#endif
#endif
#endif
#endif

    return (p);
}

P4VOID MD_shfree(ptr)
char *ptr;
{

#if defined(BALANCE) || defined(SYMMETRY) || defined(SYMMETRY_PTX)
    shfree(ptr);
#else

#if defined(TC_2000)
    heapfree(ptr);
#else

#if defined(GP_1000)
    p4_dprintf("OOPS: MD_shfree not implemented on bfly1\n");
#else

#if defined(KSR) && defined(USE_XX_SHMALLOC)
    xx_shfree(ptr);
#else

#if defined(ALLIANT) && defined(USE_XX_SHMALLOC)
    xx_shfree(ptr);
#else
#if defined(ALLIANT) && !defined(USE_XX_SHMALLOC)
    p4_dprintf("OOPS: MD_shfree not yet implemented on alliant\n");
#else	/* ALLIANT */

#if defined(MULTIMAX)
    share_free(ptr);
#else

#if defined(SYSV_IPC) && defined(USE_XX_SHMALLOC)
    xx_shfree(ptr);

#endif
#endif
#endif
#endif
#endif
#endif
#endif
#endif

}


#if defined(GP_1000)

P4BOOL simple_lock(lock)
int *lock;
{
    register short got_lock = 0;

    while(!got_lock) 
    {
	while(*lock != 0) /* wait till clear */
	    waitspin(7); /* wait 70 microsecs - atomic ops take about 60 */
	if (atomior32(lock, 1) == 0)
	    got_lock = 1;
    }
    return(TRUE);
}

#define wait_factor 13.0                /* appx 10 microseconds */
P4VOID waitspin(n)
register int n;
{
    n = (int) ((double) n * wait_factor);      /* set wait time */
    while (n > 0)
        n--;
}

P4BOOL simple_unlock(lock)
int *lock;
{
    atomand32(lock, 0);
    return(TRUE);
}

#endif


#if defined(TC_2000)

/*
 * simple_lock()
 * This function spins on a "semaphore" (lock location).
 * It spins until we get the requested semaphore.  
 */
P4BOOL simple_lock(lock)
int *lock;
{
    register short got_lock = 0;
    while (!got_lock)
    {
	while (*lock != 0)	/* wait till clear */
	    /* wait 70 microsecs - atomic ops take about 60 */
	    waitspin(7);
	/*
	 * someone else may grab lock before we can - be sure to check what
	 * the atomic op says WAS there
	 */
	if (xmemi(lock, 1) == 0)
	    got_lock = 1;
    }
    return (TRUE);
}

#define wait_factor 13.0	/* appx 10 microseconds */
P4VOID waitspin(n)
register int n;
{
    n = (int) ((double) n * wait_factor);	/* set wait time */
    while (n > 0)
	n--;
}

/*
 * simple_unlock()
 * This function releases the designated semaphore in the set.
 * It is called after a process leaves its critical section.
 */
P4BOOL simple_unlock(lock)
int *lock;
{
    *lock = 0;
    return (TRUE);
}

#endif


#if defined(ALLIANT) && !defined(USE_XX_SHMALLOC)
/*
 * xx_malloc is a memory allocation routine.  It is called in three ways: if typ
 * == 0, then n is the number of bytes to claim as globally-shared memory
 * (from which routines can ask for shared memory).  In this case the routine
 * returns the address of the allocated block (NULL, if an allocation failure
 * occurs).
 *
 * else if typ == 1, then n is taken to be the amount of shared memory
 * requested.  In this case, the routine returns the address of a block of at
 * least n charecters in length.
 *
 * else if typ == 2, then the routine is being asked to return the address of
 * the globally- shared block of memory.
 *
 * The view of shared memory supported by xx_malloc is that a single massive chunk
 * of memory is acquired and handed out by xx_malloc calls with 1 as first
 * argument.
 */

struct mem_blk
{
    char *next;
    int l_mem;
    MD_lock_t MEM;
    int pad;			/* pad out to 8-byte boundary */
};

char *xx_malloc(typ, n)
int typ, n;
{
    static struct mem_blk *glob_mem = (struct mem_blk *) NULL;
    static int l_mem = 0;
    char *rc;
    int i;
    char *c;
    /* alliant stuff */
    long id;
    /* end alliant stuff */

    switch (typ)
    {
      case 0:			/* initialize */
	/* pad & malloc */

	/* round to mult of pg size before allocing on page boundary */
	n = ((n + 4095) / 4096) * 4096;

	if ((glob_mem = (struct mem_blk *) valloc((long) n)) == NULL)
	    p4_error("xx_malloc: failed in valloc", n);

	id = (long) getpid();	/* get a unique id */

	if (create_shared_region(id, (char *) glob_mem, (long) n, 0))
	    p4_error("xx_malloc: failed in create_shared_region", n);

	glob_mem->next = (char *) (glob_mem + sizeof(struct mem_blk));
	glob_mem->l_mem = n;
	rc = glob_mem->next;
	MD_lock_init(&glob_mem->MEM);
	break;

      case 1:
	i = (n + 7) & (~007);
	MD_lock(&glob_mem->MEM);
	if (glob_mem->l_mem < i)
	{
	    p4_dprintf("*** global allocation failure ***\n");
	    p4_dprintf("*** attempted %d bytes, %d left\n",
		       i, glob_mem->l_mem);
	    rc = NULL;
	    MD_unlock(&glob_mem->MEM);
	    p4_error("xx_malloc: global alloc failed", i);
	}
	else
	{
	    rc = glob_mem->next;
	    glob_mem->next += i;
	    glob_mem->l_mem -= i;
	    /*
	     * B printf("allocated %d bytes of shared memory\n",i);
	     * printf("at %x\n",rc); E
	     */
	}
	MD_unlock(&glob_mem->MEM);
	break;

      case 2:
	rc = (char *) glob_mem;
	break;

      default:
	printf("*** illegal call to xx_malloc *** typ=%d\n", typ);
    }
    return (rc);
}

#endif	/* ALLIANT */

#ifdef GP_1000
/*
 * xx_malloc is a memory allocation routine.  It is called in three ways: if typ
 * == 0, then n is the number of bytes to claim as globally-shared memory
 * (from which routines can ask for shared memory).  In this case the routine
 * returns the address of the allocated block (NULL, if an allocation failure
 * occurs).
 *
 * else if typ == 1, then n is taken to be the amount of shared memory
 * requested.  In this case, the routine returns the address of a block of at
 * least n charecters in length.
 *
 * else if typ == 2, then the routine is being asked to return the address of
 * the globally- shared block of memory.
 *
 * The view of shared memory supported by xx_malloc is that a single massive chunk
 * of memory is acquired and handed out by xx_malloc calls with 1 as first
 * argument.
 */

struct mem_blk
{
    char *next;
    int l_mem;
    MD_lock_t MEM;
    int pad;			/* pad out to 8-byte boundary */
};

char *xx_malloc(typ, n)
int typ, n;
{
    static struct mem_blk *glob_mem = (struct mem_blk *) NULL;
    static int l_mem = 0;
    char *rc;
    int i;
    char *c;
    /* bbn stuff */
#define SHMEM_BASE 0x401000
    vm_address_t shmem_seg;
    union cluster_status cl_stat;
    int clus_size;
    int blk_cnt, ok;
    /* end bbn stuff */

    switch (typ)
    {
      case 0:			/* initialize */
	/* pad & malloc */

	/*
	 * printf("clus_size = %d, &clus_size = %d\n",clus_size,&clus_size);
	 */
	cluster_stat(HOME_CLUSTER, GET_NODE_LIST, &cl_stat, &clus_size);
	/*
	 * printf("clus_size = %d, &clus_size = %d\n",clus_size,&clus_size);
	 */

	blk_cnt = ((n / clus_size) / vm_page_size) + 1;

	/* printf("n = %d,page size = %d  ",n,vm_page_size); */
	/*
	 * printf("clus_size = %d, blk_cnt = %d\n", clus_size,blk_cnt);
	 */

	ok = 1;
	for (i = 0; (i < clus_size) && ok; i++)
	{
	    shmem_seg = SHMEM_BASE + (vm_page_size * blk_cnt) * i;
	    if (vm_allocate_and_bind(task_self(), &shmem_seg, blk_cnt * vm_page_size,
				     FALSE, i) != KERN_SUCCESS)
	    {
		printf("vm_allocate_and_bind failed\n");
		ok = 0;
	    }
	    else if (vm_inherit(task_self(), shmem_seg, blk_cnt * vm_page_size,
				VM_INHERIT_SHARE) != KERN_SUCCESS)
	    {
		printf("vm_inherit failed\n");
		ok = 0;
	    }
	}

	if (ok)
	{			/* everything succeeded */
	    glob_mem = (struct mem_blk *) SHMEM_BASE;
	    glob_mem->next = (char *) (SHMEM_BASE + sizeof(struct mem_blk));
	    glob_mem->l_mem = n;
	    rc = glob_mem->next;
	    MD_lock_init(&glob_mem->MEM);
	}
	break;

      case 1:
	i = (n + 7) & (~007);
	MD_lock(&glob_mem->MEM);
	if (glob_mem->l_mem < i)
	{
	    p4_dprintf("*** global allocation failure ***\n");
	    p4_dprintf("*** attempted %d bytes, %d left\n",
		       i, glob_mem->l_mem);
	    rc = NULL;
	    MD_unlock(&glob_mem->MEM);
	    p4_error("xx_malloc: global alloc failed", i);
	}
	else
	{
	    rc = glob_mem->next;
	    glob_mem->next += i;
	    glob_mem->l_mem -= i;
	    /* printf("allocated %d bytes of shared memory at %x\n",i,rc); */
	}
	MD_unlock(&glob_mem->MEM);
	break;

      case 2:
	rc = (char *) glob_mem;
	break;

      default:
	printf("*** illegal call to xx_malloc *** typ=%d\n", typ);
    }
    return (rc);
}

#endif

#if defined(TC_2000)

/*
 * xx_malloc is a memory allocation routine.  It is called in two ways:
 * if typ == 1, then n is taken to be the amount of shared memory
 * requested.  In this case, the routine returns the address of a block of at
 * least n charecters in length.  We round up to cache-line size, since we
 * may have specified cache attributes....
 *
 * else if typ == 2, then n is interpreted as a character pointer, pointing to
 * a string containing a filename that is used for rendesvous as a
 * memory-mapped-file, thus allowing unrelated processes to allocate shared
 * memory with each other.
 *
 * else if typ == 3, then heapsync() is called to help processes avoid doing
 * shared memory map-ins at reference time.
 *
 */

char *xx_malloc(typ, n)
int typ, n;
{
    char *mem_ptr, *mapped_filename;
    int alloc_chunk;

    switch (typ)
    {

      case 1:
	/* pad to multiple of cache-line size */
	alloc_chunk = (n + 17) & (~017);
	/*
	 * B printf("allocated %d bytes of shared memory\n",i); printf("at
	 * %x\n",mem_ptr); E
	 */
	/* gag - refer to our global at the top of the file */
	if ((mem_ptr = heapmalloc(characteristic, locality, alloc_chunk)) == NULL)
	    p4_dprintf("*** global allocation failure - general ***\n");
	/* reset hints to "defaults" */
	characteristic = HEAP_INTERLEAVED;
	locality = HEAP_ANYWHERE;	/* $$$ should be HEAP_SCATTERED */
	break;

      case 2:
	heapfile((char *) n);
	break;

      case 3:
	heapsync();
	break;

      default:
	printf("*** illegal call to xx_malloc *** typ=%d\n", typ);
    }
    return (mem_ptr);
}
#endif


#if defined(IPSC860)

struct p4_msg *MD_i860_recv()
/* Low level ipsc 860 message receive routine.
 *
 * All messages should be of the p4_msg type.
 *
 * Blocks until it receives a message.
 *
 * If the type is "ACK_REQUEST", then send back and "ACK_REPLY", to
 * confirm that the message was received.  If the message is of this
 * type, than it is sent in the struct tmsg form, to encapsulate the
 * user's chosen type of message, which gets ignored in this routine.
 */
{
    long type;
    int proc = NODE_PID, node, alloc_size, msg_size;
    struct p4_msg *m;		/* WARNING: deallocate above  */
    char ack = 'a';

    /*
     * Probe to see how big the incoming message is.  Block until that
     * message comes in.  Allocate it, and receive it.  All the p4_msg
     * information should automatically, since the message is a p4_msg type.
     */
    p4_dprintfl(20, "receiving a msg via i860 crecv\n");
    cprobe(ANY_P4TYPE_IPSC);
    alloc_size = (int) infocount();
    type = (int) infotype();
    node = (int) infonode();
    msg_size = alloc_size - sizeof(struct p4_msg) + sizeof(char *);
    m = alloc_p4_msg(msg_size);
    crecv(ANY_P4TYPE_IPSC, (char *) m, (long) alloc_size);
    p4_dprintfl(10, "received msg via i860 crecv from=%d type=%d \n",m->from,m->type);

    /*
     * If the type is "ACK_REQUEST", fire off the reply. If sender was the
     * host, send it to the procid of the host.
     */
    if (type == ACK_REQUEST_IPSC)
    {
	p4_dprintfl(30, "sending ack to %d\n", m->from);
	csend(ACK_REPLY_IPSC, &ack, sizeof(char), node, proc);
	p4_dprintfl(30, "sent ack to %d\n", m->from);
    }
    return (m);
}


int MD_i860_send(m)
struct p4_msg *m;
/*
 * Send the message, nonblocking, no wait for acknowledgement of receipt.
 */
{
    int proc = NODE_PID, to;
    char buf;			/* buffer for the ack message */
    int len;

    to = p4_local->conntab[m->to].port;

    p4_dprintfl(20, "sending msg of type %d from %d to %d via i860 send\n",m->type,m->from,m->to);
    len = m->len + sizeof(struct p4_msg) - sizeof(char *);
    if (!(m->ack_req & P4_ACK_REQ_MASK))
    {
	m->msg_id = (int) isend((long) NO_TYPE_IPSC, m, (long) len, (long) to, (long) proc);
	(p4_global->cube_msgs_out)++;
	p4_dprintfl(10, "sent msg of type %d from %d to %d via i860 isend\n",m->type,m->from,m->to);
    }
    else
    {
	/* Send a message, asking for an acknowledgement of receipt. */
	csend((long) ACK_REQUEST_IPSC, m, (long) len, (long) to, (long) proc);
	m->msg_id = -1;		/* already waited for by csend */
	/* Wait for the acknowledgement. */
	p4_dprintfl(30, "waiting for ack from %d\n", m->to);
	crecv((long) ACK_REPLY_IPSC, &buf, (long) sizeof(char));
	p4_dprintfl(30, "received ack from %d\n", m->to);
	p4_dprintfl(10, "sent msg of type %d from %d to %d via i860 csend\n",m->type,m->from,m->to);
    }
}

P4BOOL MD_i860_msgs_available()
{
    P4BOOL rc;

    rc = (P4BOOL) iprobe(ANY_P4TYPE_IPSC);
    return (rc);
}


/* endif for ipsc860 */
#endif



#if defined(CM5)

struct p4_msg *MD_CM5_recv()
/* Low level CM-5 message receive routine.
 *
 * All messages should be of the p4_msg type.
 *
 * Blocks until it receives a message.
 *
 * If the type is "ACK_REQUEST", then send back and "ACK_REPLY", to
 * confirm that the message was received.  If the message is of this
 * type, than it is sent in the struct tmsg form, to encapsulate the
 * user's chosen type of message, which gets ignored in this routine.
 */
{
    int type, node, alloc_size, msg_size;
    struct p4_msg *m;		/* WARNING: deallocate above  */
    char ack = 'a';

    /*
     * Probe to see how big the incoming message is.  Block until that
     * message comes in.  Allocate it, and receive it.  All the p4_msg
     * information should automatically, since the message is a p4_msg type.
     */
    p4_dprintfl(20, "receiving a msg via cm-5 recv\n");
    CMMD_msg_pending(CMMD_ANY_NODE, CMMD_ANY_TAG);
    alloc_size = CMMD_bytes_received();
    type = CMMD_msg_tag();
    node = CMMD_msg_sender();
    msg_size = alloc_size - sizeof(struct p4_msg) + sizeof(char *);
    m = alloc_p4_msg(msg_size);
    CMMD_receive(node, type, (void *) m, alloc_size);
    p4_dprintfl(10, "received msg via cm-5 recv from=%d type=%d \n",m->from,m->type);

    if (type == ACK_REQUEST_CM5)
    {
	p4_dprintfl(30, "sending ack to %d\n", m->from);
	CMMD_send_noblock(m->from,ACK_REPLY_CM5,&ack,sizeof(char));
	p4_dprintfl(30, "sent ack to %d\n", m->from);
    }
    return (m);
}


int MD_CM5_send(m)
struct p4_msg *m;
{
    int to,len;
    char buf;			/* buffer for the ack message */

    to = p4_local->conntab[m->to].port;

    p4_dprintfl(20, "sending msg of type %d from %d to %d via cm5 send\n",m->type,m->from,m->to);
    len = m->len + sizeof(struct p4_msg) - sizeof(char *);
    if (!(m->ack_req & P4_ACK_REQ_MASK))
    {
	CMMD_send_noblock(to, NO_TYPE_CM5, (void *) m, len);
	p4_dprintfl(10, "sent msg of type %d from %d to %d via cm5 send\n",m->type,m->from,m->to);
    }
    else
    {
	/* Send a message, asking for an acknowledgement of receipt. */
	CMMD_send_noblock(to, ACK_REQUEST_CM5, (void *) m, len);
	/* Wait for the acknowledgement. */
	p4_dprintfl(30, "waiting for ack from %d\n", m->to);
        CMMD_receive(to, ACK_REPLY_CM5, (void *) &buf, sizeof(char));
	p4_dprintfl(30, "received ack from %d\n", m->to);
	p4_dprintfl(10, "sent msg of type %d from %d to %d via cm5 csend\n",m->type,m->from,m->to);
    }
}

P4BOOL MD_CM5_msgs_available()
{
    P4BOOL rc;

    rc = CMMD_msg_pending(CMMD_ANY_NODE,CMMD_ANY_TAG);
    return (rc);
}

/* endif for cm5 */
#endif


#if defined(IPSC860)  ||  defined(CM5)

#if defined(IPSC860)
#define MYNODE mynode
#endif
#if defined(CM5)
#define MYNODE CMMD_self_address
#endif


int ns_start(argc, argv)
int *argc;
char **argv;
{
    char *s;
    char ns_host[100];
    struct bm_rm_msg bm_msg;

    sprintf(whoami, "ns_%d_%d", MYNODE(), getpid());

    /* Send off my info to my rm for forwarding to bm */
    bm_msg.type = p4_i_to_n(REMOTE_SLAVE_INFO);
    bm_msg.slave_idx = p4_i_to_n(MYNODE());
    bm_msg.slave_pid = p4_i_to_n(getpid());
    bm_msg.switch_port = p4_i_to_n(-1);
    ns_host[0] = '\0';
    get_qualified_hostname(ns_host);
    strcpy(bm_msg.host_name,ns_host);

#   if defined(IPSC860)
    crecv(SYNC_MSG, &bm_msg, (long) sizeof(struct bm_rm_msg));
    csend((long) INITIAL_INFO, &bm_msg, (long) sizeof(struct bm_rm_msg), (long) 0, (long) NODE_PID);
    crecv(INITIAL_INFO, &bm_msg, (long) sizeof(struct bm_rm_msg));
#   endif 
#   if defined(CM5)
    CMMD_receive(CMMD_ANY_NODE, CMMD_ANY_TAG, (void *) &bm_msg, sizeof(struct bm_rm_msg));
    if (CMMD_msg_tag() != SYNC_MSG)
        exit(0);
    CMMD_send_noblock(0, INITIAL_INFO, &bm_msg, sizeof(struct bm_rm_msg));
    CMMD_receive(CMMD_ANY_NODE, INITIAL_INFO, (void *) &bm_msg, sizeof(struct bm_rm_msg));
#   endif 

    if (strcmp(bm_msg.version,p4_version()) != 0)
    {
	p4_dprintf("my version is %s\n",p4_version());
	p4_error("version does not match master \n",0);
    }
    if ((s = (char *) rindex(bm_msg.pgm,'/'))  ==  NULL)
    {
	p4_dprintf("my fullpathname is invalid: %s\n",bm_msg.pgm);
	p4_error("fullpathname is invalid \n",0);
    }
    else
    {
	*s = '\0';  /* chg to directory name only */
	chdir(bm_msg.pgm);
    }
    globmemsize = p4_n_to_i(bm_msg.memsize);
    logging_flag = p4_n_to_i(bm_msg.logging_flag);
    if (logging_flag)
	ALOG_ENABLE;
    else
	ALOG_DISABLE;

    MD_initmem(globmemsize);
    alloc_global();  /* sets p4_global */
    p4_local = alloc_local_rm();
    p4_global->num_in_proctable = p4_n_to_i(bm_msg.numinproctab);
    p4_global->local_slave_count = p4_n_to_i(bm_msg.numslaves);
    debug_level = p4_n_to_i(bm_msg.debug_level);
    strcpy(p4_global->application_id, bm_msg.application_id);

#   if defined(IPSC860)
    crecv(INITIAL_INFO, p4_global->proctable, (long) sizeof(p4_global->proctable));
#   endif 
#   if defined(CM5)
    CMMD_receive(CMMD_ANY_NODE, INITIAL_INFO, (void *) p4_global->proctable, sizeof(p4_global->proctable));
#   endif 

    p4_local = alloc_local_slave();
    p4_local->listener_fd = -1;
    p4_local->my_id = p4_get_my_id_from_proc();
    sprintf(whoami, "p%d_%d", p4_get_my_id(), getpid());

    setup_conntab();

    usc_init();
    ALOG_SETUP(p4_local->my_id,ALOG_TRUNCATE);
    ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");

    slave();
    ALOG_LOG(p4_local->my_id,END_USER,0,"");
    ALOG_OUTPUT;
    exit(0);
}

/* endif for ifdef ipsc860 or cm5 */
#endif 


int MD_clock()
{
    int i;

#if defined(SYMMETRY_PTX)
    time_t t;

    time(&t);
    i = (int) (t - p4_global->reference_time);
    i = i * 1000;
#endif

#if defined(SUN)   || defined(RS6000)     || defined(DEC5000) \
 || defined(NEXT)  || defined(KSR)        || defined(CM5)     \
 || defined(SYMMETRY) || defined(BALANCE)                     \
 || defined(GP_1000)  || defined(TC_2000) || defined(CRAY)    \
 || defined(TITAN)    || defined(ALLIANT) || defined(SGI)     \
 || defined(MULTIMAX) || defined(IBM3090) || defined(HP)
    struct timeval tp;
    struct timezone tzp;

    gettimeofday(&tp, &tzp);
    i = (int) (tp.tv_sec - p4_global->reference_time);
    i *= 1000;
    i += (int) (tp.tv_usec / 1000);
#endif

#if defined(IPSC860)
    i = (int) (mclock() - p4_global->reference_time);
#endif

    return (i);
}

#ifdef DELTA
int myhost()
{
    return (0);
}
#endif


#ifdef SYSV_IPC

int init_sysv_semset(setnum)
int setnum;
{
    int i, semid;
#   if defined(IBM3090) || defined(RS6000) ||    \
       defined(TITAN)  || defined(DEC5000) ||    \
       defined(HP) || defined(KSR)
    int arg;
#   else
    union semun arg;
#   endif

#   if defined(IBM3090) || defined(RS6000) ||    \
       defined(TITAN)  || defined(DEC5000) ||    \
       defined(HP) || defined(KSR)
    arg = 1;
#   else
    arg.val = 1;
#   endif

    if ((semid = semget(getpid()+setnum,10,IPC_CREAT|0600)) < 0)
    {
	p4_error("semget failed for setnum=%d\n",setnum);
    }
    for (i=0; i < 10; i++)
    {
	if (semctl(semid,i,SETVAL,arg) == -1)
	{
	    p4_error("semctl setval failed\n",-1);
	}
    }
    return(semid);
}

P4VOID MD_lock_init(L)
MD_lock_t *L;
{
int setnum;

    MD_lock(&(p4_global->slave_lock));
    setnum = p4_global->sysv_next_lock / 10;
    if (setnum > P4_MAX_SYSV_SEMIDS)
    {
	p4_error("exceeding max num of p4 semids\n",P4_MAX_SYSV_SEMIDS);
    }
    if (p4_global->sysv_next_lock % 10 == 0)
    {
	p4_global->sysv_semid[setnum] = init_sysv_semset(setnum);
	p4_global->sysv_num_semids++;
    }
    L->semid  = p4_global->sysv_semid[setnum];
    L->semnum = p4_global->sysv_next_lock - (setnum * 10);
    p4_global->sysv_next_lock++;
    MD_unlock(&(p4_global->slave_lock));
}


P4VOID MD_lock(L)
MD_lock_t *L;
{
    sem_lock[0].sem_num = L->semnum;
    if (semop(L->semid,&sem_lock[0],1) < 0)
    {
        p4_error("OOPS: semop lock failed\n",*L);
    }
}

P4VOID MD_unlock(L)
MD_lock_t *L;
{
    sem_unlock[0].sem_num = L->semnum;
    if (semop(L->semid,&sem_unlock[0],1) < 0)
    {
        p4_error("OOPS: semop unlock failed\n",L);
    }
}
#endif

#if defined(TCMP)

P4BOOL MD_tcmp_msgs_available(req_type,req_from)
int *req_from,*req_type;
{
    char *msg = NULL;
    int len_rcvd;
    tcmpfunp matcher;
    tcmp_status tcmpstat;

    if (*req_type == -1)
	if (*req_from == -1)
	    matcher = TCMP_MATCH_ANY;
	else
	    matcher = TCMP_MATCH_SENDER;
    else if (*req_from == -1)
	matcher = TCMP_MATCH_TYPE;
    else
	matcher = TCMP_MATCH_BOTH;
    tcmpstat = tcmp_receive(matcher, req_from, req_type,
			    TCMP_NOBLOCK | TCMP_NOCOPY | TCMP_NODEQUEUE,
			    &len_rcvd, &msg);
    if (tcmpstat == TCMP_SUCCESS)
	return(TRUE);
    else
	return(FALSE);
}

int MD_tcmp_send(type, from, to, msg, len, data_type, ack_req)
int type, from, to, len, data_type, ack_req;
char *msg;
{
    int sendflags;
    tcmp_status tcmpstat;

    if (ack_req & P4_ACK_REQ_MASK)
	sendflags = 0;
    else
	sendflags = TCMP_NOBLOCK;
    tcmpstat = tcmp_send(to, type, sendflags, len, msg);
    if (tcmpstat != TCMP_SUCCESS)
	p4_error("bad status %d on tcmp_send",tcmpstat);
    return(0);
}

struct p4_msg *MD_tcmp_recv()
{
    int type, from, len;
    struct p4_msg *msg;
    tcmp_status tcmpstat;

    tcmpstat = tcmp_receive(TCMP_MATCH_ANY,&from,&type,TCMP_NOCOPY,&len,&msg);
    if (tcmpstat != TCMP_SUCCESS)
	p4_error("bad tcmp status %d on receive", tcmpstat);

    return (msg);
}

#endif

int data_representation(machine_type)
char *machine_type;
{
    if (strcmp(machine_type, "SUN") == 0)             return 1;
    if (strcmp(machine_type, "HP") == 0)              return 1;
    if (strcmp(machine_type, "RS6000") == 0)          return 1;
    if (strcmp(machine_type, "SGI") == 0)             return 1;
    if (strcmp(machine_type, "NEXT") == 0)            return 1;
    if (strcmp(machine_type, "CM5") == 0)             return 1;
    if (strcmp(machine_type, "SYMMETRY") == 0)        return 2;
    if (strcmp(machine_type, "SYMMETRY_PTX") == 0)    return 2;
    if (strcmp(machine_type, "SUN386I") == 0)         return 2;
    if (strcmp(machine_type, "DEC5000") == 0)         return 3;
    if (strcmp(machine_type, "IBM3090") == 0)         return 4;
    if (strcmp(machine_type, "TITAN") == 0)           return 5;
    if (strcmp(machine_type, "FX8") == 0)             return 6;
    if (strcmp(machine_type, "FX2800") == 0)          return 7;
    if (strcmp(machine_type, "FX2800_SWITCH") == 0)   return 7;
    if (strcmp(machine_type, "IPSC860") == 0)         return 8;
    if (strcmp(machine_type, "IPSC860_SOCKETS") == 0) return 8;
    if (strcmp(machine_type, "DELTA") == 0)           return 8;
    if (strcmp(machine_type, "BALANCE") == 0)         return 12;
    if (strcmp(machine_type, "MULTIMAX") == 0)        return 15;
    if (strcmp(machine_type, "CRAY") == 0)            return 16;
    if (strcmp(machine_type, "GP_1000") == 0)         return 17;
    if (strcmp(machine_type, "TC_2000") == 0)         return 18;
    if (strcmp(machine_type, "TC_2000_TCMP") == 0)    return 18;
    if (strcmp(machine_type, "KSR") == 0)             return 19;
}
SHAR_EOF
if test -f 'p4_MD.h'
then
	echo shar: over-writing existing file "'p4_MD.h'"
fi
cat << \SHAR_EOF > 'p4_MD.h'

/* ------------------ Machine Dependent Definitions ---------------- */
/*
        It is important to maintain the order of many of the 
        definitions in this file.
*/


#ifdef TC_2000_TCMP
#define TC_2000
#define TCMP
#endif

#if defined(FX8)  ||  defined(FX2800)  || defined(FX2800_SWITCH)
#define ALLIANT
#endif

#if defined(FX2800)  || defined(FX2800_SWITCH)
#define VPRINTF
#endif


#if defined(DELTA) || defined(IPSC860_SOCKETS)
#define IPSC860
#endif

#if defined(NEXT)  || defined(KSR) ||  defined(IPSC860)
#define GLOBAL
#endif


#if defined(SUN)        || defined(DEC5000)  || \
    defined(NEXT)       || defined(KSR)      || \
    defined(SYMMETRY)   || defined(BALANCE)  || \
    defined(ALLIANT)    || defined(MULTIMAX) ||  defined(CM5) || \
    defined(GP_1000)    || defined(TC_2000)  ||  defined(IBM3090)

#define P4BSD

#endif

#if defined(SUN)        || defined(DEC5000)  || \
    defined(NEXT)       || defined(KSR)      || \
    defined(SYMMETRY)   || defined(BALANCE)  || \
    defined(ALLIANT)    || defined(MULTIMAX) || \
    defined(GP_1000)    || defined(TC_2000)  ||  defined(IBM3090)

#define CAN_DO_SETSOCKOPT

#endif

#if defined(RS6000)          ||                          \
    defined(IPSC860_SOCKETS) ||                         \
    defined(DELTA)           || defined(TITAN)        || \
    defined(SGI)             || defined(CRAY)         || \
    defined(HP)              || defined(SYMMETRY_PTX)

#define CAN_DO_SETSOCKOPT

#endif


#if defined(RS6000)       || \
    defined(IPSC860)      ||                          \
    defined(DELTA)        || defined(TITAN)        || \
    defined(SGI)          || defined(CRAY)         || \
    defined(HP)           || defined(SYMMETRY_PTX)

#define P4SYSV

#endif


#ifdef P4SYSV
#define P4_SYSV_SHM_SEGSIZE (1*1024*1024)
#endif


#ifdef P4SYSV
#define SIGNAL_P4 sigset
#else
#define SIGNAL_P4 signal
#endif


#ifndef P4BOOL
#define P4BOOL int
#endif

#if defined(BALANCE)  ||  defined(FX8)
#define P4VOID int
#else 
#define P4VOID void
#endif


/*------------------ Encore Multimax ---------------------- */


#if defined(MULTIMAX)

#include <parallel.h>

#ifndef LINT
typedef LOCK *MD_lock_t;
#define MD_lock_init(l)  *(l) = spin_create(PAR_UNLOCKED);
#define MD_lock(l)       spin_lock(*(l));
#define MD_unlock(l)     spin_unlock(*(l));
#endif

#define GLOBMEMSIZE  (2*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS
#define P4_MAX_MSG_QUEUES 64

#endif


/*------------------ Sequent Balance or Symmetry ---------------------- */


#if !defined(SYMMETRY) && !defined(SYMMETRY_PTX) && !defined(BALANCE)
#define CAN_HANDLE_SIGSEGV
#endif

#if defined(BALANCE) || defined(SYMMETRY) || defined(SYMMETRY_PTX)

#include <parallel/parallel.h>

typedef slock_t MD_lock_t;

#ifndef LINT
#define MD_lock_init(l)  s_init_lock(l);
#define MD_lock(l)       s_lock(l);
#define MD_unlock(l)     s_unlock(l);
#endif
extern char *shmalloc();
#if defined(SYMMETRY_PTX)
extern P4VOID *malloc();
#else
extern char *malloc();
#endif

#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS
#define P4_MAX_MSG_QUEUES 64

#endif

/*---------------------------- Alliant -------------------------------- */
#if defined(ALLIANT)

typedef char MD_lock_t;

#ifndef LINT
#define MD_lock_init(l)  initialize_lock(l);
#define MD_lock(l)       lock(l);
#define MD_unlock(l)     unlock(l);
#endif
extern char *valloc();

#ifdef FX8
#define GLOBMEMSIZE  (2*1024*1024)    
#else
#define GLOBMEMSIZE  (8*1024*1024)    /* Lots of memory ... use it! */
#endif

#define USE_XX_SHMALLOC          /* If not defined uses dumb xx_malloc */

#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS
#define P4_MAX_MSG_QUEUES 64

#endif

#if defined(FX2800_SWITCH)
#include "sw.h"
#define CAN_DO_SWITCH_MSGS
#endif


/*---------------------------- Uniprocessors -------------------------- */

#if defined(CRAY) || defined(NEXT)
#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define P4_MAX_MSG_QUEUES 1
typedef int MD_lock_t;
#define MD_lock_init(l)
#define MD_lock(l)
#define MD_unlock(l)
#endif

#if defined(SUN) ||  defined(DEC5000)      \
    || defined(RS6000) || defined(IBM3090) \
    || defined(TITAN) || defined(SGI)      \
    || defined(HP)

#if defined(SYSV_IPC)
#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS
#define USE_XX_SHMALLOC
#define P4_MAX_MSG_QUEUES 4
#define P4_MAX_SYSV_SHMIDS  8
#define P4_MAX_SYSV_SEMIDS  8

typedef struct { int semid;  int semnum; }   MD_lock_t;

#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/sem.h>

static struct sembuf sem_lock[1] = {
    0, -1, 0
};
static struct sembuf sem_unlock[1] = {
    0, 1, 0
};

#else

#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define P4_MAX_MSG_QUEUES 1
typedef int MD_lock_t;
#define MD_lock_init(l)
#define MD_lock(l)
#define MD_unlock(l)

#endif
#endif



/*---------------------------- IPSC860 Cube --------------------------- */

#if defined(IPSC860)

#    if defined(DELTA)
#        define P4_MAX_CUBE_MSGS_OUT 5
#    else
#        define P4_MAX_CUBE_MSGS_OUT 5
#    endif

#define MD_cube_send  MD_i860_send
#define MD_cube_recv  MD_i860_recv
#define MD_cube_msgs_available  MD_i860_msgs_available

typedef int MD_lock_t;

#ifndef LINT
#define MD_lock_init(l)
#define MD_lock(l)
#define MD_unlock(l)
#endif

#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_CUBE_MSGS
#define P4_MAX_MSG_QUEUES 1

#define ALL_NODES -1

#define NO_TYPE_IPSC     0 
#define ACK_REQUEST_IPSC 1
#define ACK_REPLY_IPSC   2
#define ANY_P4TYPE_IPSC    0x80000007

#define NODE_PID 0

#if defined(IPSC860_SOCKETS)
#define CAN_DO_SOCKET_MSGS
/*****
#include <CMC/sys/types.h>
#include <CMC/sys/socket.h>
#include <CMC/netinet/in.h>
#include <CMC/netdb.h>
*****/
#include <CMC/ntoh.h>
#endif

#endif    

/*---------------------------- CM-5 --------------------------- */

#if defined(CM5)

#include <cm/cmmd.h>
#include <cm/cmmd-io.h>

typedef int MD_lock_t;

#ifndef LINT
#define MD_lock_init(l)
#define MD_lock(l)
#define MD_unlock(l)
#endif

#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_CUBE_MSGS
#define P4_MAX_MSG_QUEUES 1

#define NO_TYPE_CM5     0 
#define ACK_REQUEST_CM5 1
#define ACK_REPLY_CM5   2
#define ANY_P4TYPE_CM5    CMMD_ANY_TAG

#define MD_cube_send  MD_CM5_send
#define MD_cube_recv  MD_CM5_recv
#define MD_cube_msgs_available  MD_CM5_msgs_available

#endif    

/*----------------   KSR             -------------------------*/
#if defined(KSR)
#include <sys/mman.h>
#include <pthread.h>

#define USE_XX_SHMALLOC
#define GLOBMEMSIZE  (8*1024*1024)
#define P4_MAX_MSG_QUEUES 32
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS

#define MD_lock_t       msemaphore
#define MD_lock_init(l) msem_init(l, MSEM_UNLOCKED)
#define MD_lock(l)      msem_lock(l, 0)
#define MD_unlock(l )   msem_unlock(l, 0)

#endif



/*------------------ Butterfly TC-2000/GP-1000 -------------- */
#if defined(TC_2000)  ||  defined(GP_1000)
#include <mach.h>    
#include <sys/cluster.h>
#include <sys/kern_return.h>
#include <heap.h>

char *xx_malloc();
P4VOID MD_malloc_hint();

#ifdef MALLOC_STATS
static unsigned int allocated = 0;
#endif

#define MD_lock_t       int
#ifndef LINT
#define MD_lock_init(l) simple_unlock(l)
#define MD_lock(l)      simple_lock(l)
#define MD_unlock(l)    simple_unlock(l)
#endif

#define GLOBMEMSIZE  (1*1024*1024)
#define CAN_DO_SOCKET_MSGS
#define CAN_DO_XDR
#define CAN_DO_SHMEM_MSGS
#define P4_MAX_MSG_QUEUES 128

#endif

#ifdef TCMP
#define CAN_DO_TCMP_MSGS
/* #include </Net/sparky/sparky1/lusk/lepido/tcmp/tcmp.h> */
#include </usr/bbnm/tcmp/tcmp.h>
#endif

/* ----------------- Can be made machine dependent -------------------*/

typedef unsigned long p4_usc_time_t;


extern P4VOID exit();

#define P4_MAXPROCS 128

#ifndef LINT
#define p4_malloc malloc
#define p4_free free
#define p4_clock MD_clock
#endif
SHAR_EOF
if test -f 'p4_alloc.c'
then
	echo shar: over-writing existing file "'p4_alloc.c'"
fi
cat << \SHAR_EOF > 'p4_alloc.c'
#include "p4.h"
#include "p4_sys.h"

struct local_data *alloc_local_bm()
{
    struct local_data *l;

    l = (struct local_data *) p4_malloc(sizeof(struct local_data));
    if (l == NULL)
    {
	p4_dprintf("OOPS: alloc_local_bm: p4_malloc failed \n");
	return (l);
    }

    l->am_bm = TRUE;
    l->listener_fd = -1;
    l->my_id = -1;
    l->procgroup = NULL;
    l->queued_messages = (struct p4_msg_queue *)
	p4_malloc(sizeof(struct p4_msg_queue));
    initialize_msg_queue(l->queued_messages);
    l->soft_errors = 0;

#   ifdef CAN_DO_XDR
    if ((l->xdr_buff = (char *) p4_malloc(XDR_BUFF_LEN)) == NULL)
    {
	p4_error("OOPS: alloc_local_bm: unable to malloc xdr_buff\n",NULL);
    }
    xdrmem_create(&(l->xdr_enc), l->xdr_buff, XDR_BUFF_LEN, XDR_ENCODE);
    xdrmem_create(&(l->xdr_dec), l->xdr_buff, XDR_BUFF_LEN, XDR_DECODE);
#   endif

    return (l);
}

struct local_data *alloc_local_rm()
{
    struct local_data *l;

    l = (struct local_data *) p4_malloc(sizeof(struct local_data));
    if (l == NULL)
    {
	p4_dprintf("OOPS: alloc_local_rm: p4_malloc failed \n");
	return (l);
    }

    l->am_bm = FALSE;
    l->listener_fd = -1;
    l->my_id = -1;
    l->procgroup = NULL;
    l->queued_messages = (struct p4_msg_queue *)
	p4_malloc(sizeof(struct p4_msg_queue));
    initialize_msg_queue(l->queued_messages);
    l->soft_errors = 0;

#   ifdef CAN_DO_XDR
    if ((l->xdr_buff = (char *) p4_malloc(XDR_BUFF_LEN)) == NULL)
    {
	p4_error("OOPS: alloc_local_rm: unable to malloc xdr_buff\n",NULL);
    }
    xdrmem_create(&(l->xdr_enc), l->xdr_buff, XDR_BUFF_LEN, XDR_ENCODE);
    xdrmem_create(&(l->xdr_dec), l->xdr_buff, XDR_BUFF_LEN, XDR_DECODE);
#   endif

    return (l);
}				/* alloc_local_rm */

struct local_data *alloc_local_listener()
{
    struct local_data *l;

    l = (struct local_data *) p4_malloc(sizeof(struct local_data));

    l->am_bm = FALSE;
    l->listener_fd = -1;
    l->my_id = LISTENER_ID;
    l->procgroup = NULL;
    l->queued_messages = NULL;
    l->xdr_buff = NULL;
    l->soft_errors = 0;
    return (l);
}				/* alloc_local_listener */

struct local_data *alloc_local_slave()
{
    struct local_data *l;

    l = (struct local_data *) p4_malloc(sizeof(struct local_data));

    l->am_bm = FALSE;
    l->listener_fd = -1;
    l->my_id = -1;
    l->procgroup = NULL;
    l->queued_messages = (struct p4_msg_queue *)
	p4_malloc(sizeof(struct p4_msg_queue));
    initialize_msg_queue(l->queued_messages);
    l->soft_errors = 0;

#   ifdef CAN_DO_XDR
    if (!(p4_global->local_communication_only))
    {
        if ((l->xdr_buff = (char *) p4_malloc(XDR_BUFF_LEN)) == NULL)
        {
	    p4_error("OOPS: alloc_local_slave: unable to malloc xdr_buff\n",NULL);
        }
        xdrmem_create(&(l->xdr_enc), l->xdr_buff, XDR_BUFF_LEN, XDR_ENCODE);
        xdrmem_create(&(l->xdr_dec), l->xdr_buff, XDR_BUFF_LEN, XDR_DECODE);
    }
#   endif

    return (l);
}

/*
    This routine should be called before any sends and receives are done by
    the user.  If not, some buffers may be lost.
*/
P4VOID p4_set_avail_buff(bufidx,size)
int bufidx;
int size;
{
    p4_global->avail_buffs[bufidx].size = size;
    p4_global->avail_buffs[bufidx].buff = NULL;
}


P4VOID init_avail_buffs()
{
    static int sizes[NUMAVAILS] = 
            {64,256,1024,4096,16384,65536,262144,1048576};
    int i;

    for (i = 0; i < NUMAVAILS; i++)
    {
        p4_global->avail_buffs[i].size = sizes[i];
        p4_global->avail_buffs[i].buff = NULL;
    }
}

P4VOID p4_print_avail_buffs()
{
    int i, count;
    struct p4_msg *next;

    p4_dprintf("avail lists for message buffers:\n");
    p4_lock(&p4_global->avail_buffs_lock);
    for (i = 0; i < NUMAVAILS; i++)
    {
	count = 0;
	for (next = p4_global->avail_buffs[i].buff; next; next = next->link)
	     count++;
	p4_dprintf("%d buffers of size %d\n",
		   count, p4_global->avail_buffs[i].size);
    }
    p4_unlock(&p4_global->avail_buffs_lock);
}

struct p4_msg *alloc_p4_msg(msglen)
int msglen;
{
    struct p4_msg *rmsg = NULL, **trailer;
    int i, rounded;
    P4BOOL found;

    p4_dprintfl(40, "allocating a buffer for message of size %d\n", msglen);

#   if defined(TCMP)

    rmsg = (struct p4_msg *) tcmp_allocate((sizeof(struct p4_msg) + msglen) - sizeof(char *));
    p4_dprintfl(40, "allocated new buffer at 0x%x for msg of size %d\n",
		rmsg, msglen);
    rmsg->len = msglen;
    return(rmsg);

#   else

    i = 0;
    while ((i < NUMAVAILS) && (msglen > p4_global->avail_buffs[i].size))
	i++;

    if (i == NUMAVAILS)		/* didn't find a big enough avail buffer */
    {
	rmsg = (struct p4_msg *)
	    p4_shmalloc((sizeof(struct p4_msg) + msglen) - sizeof(char *));
	p4_dprintfl(40, "allocated new buffer at0x%x for message size %d\n",
		    rmsg, msglen);
    }
    else
    {
	rounded = p4_global->avail_buffs[i].size;
	p4_lock(&p4_global->avail_buffs_lock);
	if (p4_global->avail_buffs[i].buff)
	{

#if defined(IPSC860)
	    rmsg = p4_global->avail_buffs[i].buff;
	    trailer = &(p4_global->avail_buffs[i].buff);
	    found = FALSE;
	    while (!found && (rmsg != NULL))
	    {
		if (rmsg->msg_id == -1)
		{
		    found = TRUE;
		}
		else if (msgdone((long) rmsg->msg_id))
		{
		    rmsg->msg_id = -1;	
		    (p4_global->cube_msgs_out)--;
		    found = TRUE;
		}
		else
		{
		    trailer = &((*trailer)->link);
		    rmsg = rmsg->link;
		}
	    }
	    if (!found && (p4_global->cube_msgs_out > P4_MAX_CUBE_MSGS_OUT))
	    {
		if ((rmsg = p4_global->avail_buffs[i].buff) != NULL)
		{
		    msgwait((long) rmsg->msg_id);
		    rmsg->msg_id = -1;	
		    (p4_global->cube_msgs_out)--;
		    found = TRUE;
		}
	    }
	    if (!found)
	    {
		rmsg = (struct p4_msg *)
		    p4_shmalloc((sizeof(struct p4_msg) + rounded) - sizeof(char *));
		p4_dprintfl(40, "allocated new buffer at 0x%x of size %d for message size %d\n", rmsg, rounded, msglen);
	    }
	    else
	    {
		*trailer = rmsg->link;
		p4_dprintfl(40, "reused a buffer of size %d for message size %d\n",
			    rounded, msglen);
	    }
#else
	    rmsg = p4_global->avail_buffs[i].buff;
	    p4_global->avail_buffs[i].buff = rmsg->link;
	    p4_dprintfl(40, "reused a buffer of size %d for message size %d\n",
			rounded, msglen);
#endif

	    p4_unlock(&p4_global->avail_buffs_lock);
	}
	else
	{
	    p4_unlock(&p4_global->avail_buffs_lock);
	    rmsg = (struct p4_msg *)
		p4_shmalloc((sizeof(struct p4_msg) + rounded) - sizeof(char *));
	    p4_dprintfl(40, "allocated new buffer at 0x%x of size %d for message size %d\n", rmsg, rounded, msglen);
	}
    }

    if ((rmsg == NULL) && !SOFTERR)
	p4_error("alloc_p4_message failed", 0);

    rmsg->len = msglen;
    return(rmsg);

#   endif
}				/* alloc_p4_msg */

P4VOID free_p4_msg(tmsg)
struct p4_msg *tmsg;
{
    int i, msglen;
    struct p4_msg *p;

    msglen = tmsg->len;
    p4_dprintfl(40, "freeing a buffer with message size %d\n", msglen);

    /* Sanity check here as bad message pointer causes havoc */

    if ((msglen < 0) || (msglen > P4_MAX_MSGLEN))
	p4_error("free_p4_msg: bad hdr: msglen out of range", msglen);

#   if defined(TCMP)
    if (tmsg)
	tcmp_deallocate(tmsg);
    return;

#   else

    i = 0;
    while ((i < NUMAVAILS) && (msglen > p4_global->avail_buffs[i].size))
	i++;

    if (i == NUMAVAILS)
    {
	/* buffer being freed is not a kept size */
	p4_shfree(tmsg);
	p4_dprintfl(40, "deallocated a buffer at %d for message size %d\n",
		    tmsg, msglen);
    }
    else
    {
	/* hook new buffer in at end of list */
	p4_lock(&p4_global->avail_buffs_lock);
	if ((p = p4_global->avail_buffs[i].buff) == NULL)
	    p4_global->avail_buffs[i].buff = tmsg;
	else
	{
	    while (p->link != NULL)
		p = p->link;
	    p->link = tmsg;
	}
	tmsg->link = NULL;
	p4_unlock(&p4_global->avail_buffs_lock);
	p4_dprintfl(40, "saved a buffer of size %d in avail list for size %d\n",
		    msglen, p4_global->avail_buffs[i].size);
    }

#   endif
}				/* free_p4_msg */

P4VOID alloc_global()
{
    int i;
    struct p4_global_data *g;

    p4_global = (struct p4_global_data *) p4_shmalloc(sizeof(struct p4_global_data));
    if (p4_global == NULL)
    {
	p4_error("alloc_global: alloc_global failed\n", sizeof(struct p4_global_data));
    }
    g = p4_global;

#ifdef SYSV_IPC
    g->slave_lock.semid = sysv_semid0;
    g->slave_lock.semnum = 1;
    g->sysv_semid[0] = sysv_semid0;
    g->sysv_num_semids = 1;
    g->sysv_next_lock = 2;  /* shmem_lock is 0 & slave_lock is 1 */
#else
    p4_lock_init(&g->slave_lock);
#endif

    g->listener_pid = -1;
    g->listener_port = -1;

    g->cube_msgs_out = 0;

    g->local_slave_count = 0;
    g->local_communication_only = TRUE;
    g->n_forked_pids = 0;

    for (i = 0; i < P4_MAX_MSG_QUEUES; i++)
    {
	initialize_msg_queue(&g->shmem_msg_queues[i]);
	g->dest_id[i] = -1;
    }

    p4_lock_init(&g->avail_buffs_lock);
    init_avail_buffs();
    p4_lock_init(&g->avail_quel_lock);
    g->avail_quel = NULL;

    g->num_in_proctable = 0;
    g->num_installed = 0;
    g->my_host_name[0] = '\0';
    get_qualified_hostname(g->my_host_name);

    p4_barrier_init(&(g->cluster_barrier));

    sprintf(g->application_id, "p4_%-8d", getpid());

#   if defined(P4BSD)
    g->max_connections = getdtablesize();
#   endif

#   if defined(P4SYSV)
#       if defined(CRAY) || defined(RS6000) || \
	   defined(IBM3090) || defined(SYMMETRY_PTX)
              g->max_connections = getdtablesize();
#       else
#       if defined(IPSC860) || defined(HP)  /* HP probably supports more */
              g->max_connections = 20;
#       else
              g->max_connections = (int) ulimit(4,0);
#       endif
#       endif
#   endif
 
}

struct listener_data *alloc_listener_info()
{
    struct listener_data *l;

    l = (struct listener_data *) p4_malloc(sizeof(struct listener_data));

    l->listening_fd = -1;
    l->slave_fd = -1;

    return (l);
}
SHAR_EOF
if test -f 'p4_args.c'
then
	echo shar: over-writing existing file "'p4_args.c'"
fi
cat << \SHAR_EOF > 'p4_args.c'
/*
 * p4_args.c   Code that looks at the arguments, recognizes any that are
 *             for p4, uses the arguments, and removes them from the
 *             command line args.
 */
#include "p4.h"
#include "p4_sys.h"

/* Macro used to see if an arg is not following the correct format. */
#define bad_arg(a)    ( ((a)==NULL) || ((*(a)) == '-') )

static char pgm[100];		/* Used to keep argv[0] for the usage cmd. */

static P4VOID usage();
static P4VOID print_version_info();
static P4VOID strip_out_args();


P4VOID process_args(argc, argv)
int *argc;
char **argv;
{
    char *s, **a;
    int c;

    /* Put the name of the called program (according to the args) into pgm */
    s = (char *)  rindex(*argv, '/');
    if (s)
	strcpy(pgm, s + 1);
    else
	strcpy(pgm, *argv);

    /* Set all command line flags to their default. */
    debug_level = 0;
    remote_debug_level = 0;
    strcpy(bm_outfile, "");
    strcpy(procgroup_file, "procgroup");
    strcpy(local_domain, "");

    /* Move to last argument, so that we can go backwards. */
    a = &argv[*argc - 1];

    /*
     * Loop backwards through arguments, catching the ones that start with
     * '-'.  Backwards is more efficient when you are stripping things out.
     */
    for (c = (*argc); c > 1; c--, a--)
    {

	if (**a != '-')
	    continue;

	if (!strcmp(*a, "-pg"))
	{
	    if (bad_arg(a[1]))
		usage();
	    strcpy(procgroup_file, a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-dbg"))
	{
	    if (bad_arg(a[1]))
		usage();
	    debug_level = atoi(a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-ssport"))
	{
	    if (bad_arg(a[1]))
		usage();
	    sserver_port = atoi(a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-rdbg"))
	{
	    if (bad_arg(a[1]))
		usage();
	    remote_debug_level = atoi(a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-gm"))
	{
	    if (bad_arg(a[1]))
		usage();
	    globmemsize = atoi(a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-dmn"))
	{
	    if (bad_arg(a[1]))
		usage();
	    strcpy(local_domain, a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-out"))
	{
	    if (bad_arg(a[1]))
		usage();
	    strcpy(bm_outfile, a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-rout"))
	{
	    if (bad_arg(a[1]))
		usage();
	    strcpy(rm_outfile_head, a[1]);
	    strip_out_args(a, argc, &c, 2);
	    continue;
	}
	if (!strcmp(*a, "-p4log"))
	{
	    strip_out_args(a, argc, &c, 1);
	    logging_flag = TRUE;
	    continue;
	}
	if (!strcmp(*a, "-p4version"))
	{
	    strip_out_args(a, argc, &c, 1);
	    print_version_info();
	    continue;
	}
	if (!strcmp(*a, "-?") || !(strcmp(*a, "-help")))
	    usage();
    }
}

static P4VOID strip_out_args(argv, argc, c, num)
char **argv;
int *argc, *c, num;
{
    char **a;
    int i;

    /* Strip out the argument. */
    for (a = argv, i = (*c); i <= *argc; i++, a++)
	*a = (*(a + num));
    (*argc) -= num;
}

static P4VOID usage()
{
    print_version_info();
    printf("p4 usage: %s options\n", pgm);
    printf("Valid options:\n");
    printf("\t-help   \tget this message\n");
    printf("\t-pg     \tprocgroup_file\n");
    printf("\t-dbg    \tset debug level\n");
    printf("\t-rdbg   \tset remote debug level\n");
    printf("\t-gm     \tset globmemsize\n");
    printf("\t-dmn    \tdomainname\n");
    printf("\t-out    \tfile \toutput file for master\n");
    printf("\t-rout   \tfile \toutput file prefix for remote masters\n");
    printf("\t-ssport \tport# private port number for secure server \n");
    printf("\t-p4log  \tenable internal p4 logging by alog; only works if p4 made with alog\n");
    printf("\t-p4version \tprint current p4 version number\n");
    printf("\n");
    exit(-1);
}

static P4VOID print_version_info()
{
        printf("\n");
        printf("p4 version number:  %s\n",p4_version());
        printf("p4  date compiled:  %s\n",P4_COMPILED_TIME);
        printf("p4   machine type:  %s\n",P4_MACHINE_TYPE);
#ifdef P4_DPRINTFL
        printf("   P4_DPRINTFL is:  on\n");
#else
        printf("   P4_DPRINTFL is:  off\n");
#endif
#ifdef ALOG_TRACE
        printf("    ALOG_TRACE is:  on\n");
#else
        printf("    ALOG_TRACE is:  off\n");
#endif
#if defined(SUN_IPC) || defined(DEC5000_IPC) || defined(RS6000_IPC) || defined(IBM3090_IPC) || \
    defined(HP_IPC)  || defined(TITAN_IPC)   || defined(SGI_IPC)
        printf("      SYSV IPC is:  on\n");
#else
        printf("      SYSV IPC is:  off\n");
#endif
        printf("\n");
}
SHAR_EOF
if test -f 'p4_bm.c'
then
	echo shar: over-writing existing file "'p4_bm.c'"
fi
cat << \SHAR_EOF > 'p4_bm.c'
#include "p4.h"
#include "p4_sys.h"

int bm_start(argc, argv)
int *argc;
char **argv;
{
    int bm_switch_port;

    sprintf(whoami, "p0_%d", getpid());
    p4_dprintfl(90,"entering bm_start\n");

    trap_sig_errs();		/* Errors can happen any time */

    logging_flag = FALSE;
    globmemsize = GLOBMEMSIZE;
    sserver_port = 753;

    process_args(argc, argv);

#   ifdef SYSV_IPC
    sysv_num_shmids = 0;
    sysv_shmid[0]  = -1;
    sysv_semid0 = init_sysv_semset(0);
#   endif

    MD_initmem(globmemsize);
    alloc_global();  /* sets p4_global */

    if (*bm_outfile)
    {
	freopen(bm_outfile, "w", stdout);
	freopen(bm_outfile, "w", stderr);
    }

    p4_local = alloc_local_bm();
    if (p4_local == NULL)
	p4_error("p4_initenv: alloc_local_bm failed\n", NULL);

    MD_initenv();
    bm_switch_port = getswport(p4_global->my_host_name);
    usc_init();

    /* big master installing himself */
    install_in_proctable(0, (-1), getpid(), p4_global->my_host_name, 
			 0, P4_MACHINE_TYPE, bm_switch_port);

    p4_local->my_id = 0;

    if (logging_flag)
	ALOG_ENABLE;
    else
	ALOG_DISABLE;

    return (0);
}

int p4_create_procgroup()
{

    p4_dprintfl(90,"entering p4_create_procgroup\n");
    if ((p4_local->procgroup = read_procgroup()) == NULL)
	return (-1);
    p4_startup(p4_local->procgroup);
}


int p4_startup(pg)
struct p4_procgroup *pg;
{
    int i, nslaves;
    int listener_port, listener_fd;
    struct bm_rm_msg bm_msg;
    struct p4_procgroup_entry *local_pg;

    p4_dprintfl(90,"entering p4_startup\n");

    if (p4_global == NULL)
	p4_error("p4 not initialized; perhaps p4_initenv not called",0);

    procgroup_to_proctable(pg);
    if (pg->num_entries > 1)
	p4_global->local_communication_only = FALSE;

#   ifdef CAN_DO_SOCKET_MSGS
    if (!p4_global->local_communication_only)
    {
	net_setup_anon_listener(10, &listener_port, &listener_fd);
	p4_global->listener_port = listener_port;
	p4_global->listener_fd = listener_fd;
	p4_dprintfl(90, "setup listener on port %d fd %d\n",
		    listener_port, listener_fd);
	p4_global->proctable[0].port = listener_port;
	SIGNAL_P4(LISTENER_ATTN_SIGNAL, handle_connection_interrupt);
    }
#   endif

    setup_conntab();

    p4_lock(&p4_global->slave_lock);
    if ((nslaves = create_bm_processes(pg)) < 0)
	return (-1);

#   ifdef CAN_DO_SOCKET_MSGS
    if (create_remote_processes(pg) < 0)
	return (-1);
#   endif

    /* let local slaves use proc table to identify themselves */
    p4_unlock(&p4_global->slave_lock); 

    send_proc_table();  /* to remote masters */

#   if defined(IPSC860)  ||  defined(CM5)
    /* send initial info and proctable to local slaves */
    /* must use p4_i_to_n procs because node slave 
       does not know if the msg is forwarded from bm */
    local_pg = &(pg->entries[0]);
    bm_msg.type = p4_i_to_n(INITIAL_INFO);
    bm_msg.numinproctab = p4_i_to_n(p4_global->num_in_proctable);
    bm_msg.numslaves = p4_i_to_n(local_pg->numslaves_in_group);
    bm_msg.debug_level = p4_i_to_n(remote_debug_level);
    bm_msg.memsize = p4_i_to_n(globmemsize);
    bm_msg.logging_flag = p4_i_to_n(logging_flag);
    strcpy(bm_msg.application_id, p4_global->application_id);
    strcpy(bm_msg.version, p4_version());
    strcpy(bm_msg.pgm, local_pg->slave_full_pathname);
    for (i = 1; i <= nslaves; i++)
    {
	p4_dprintfl(90,"sending initinfo to slave %d of %d\n",i,nslaves);
#       if defined(IPSC860)
	csend((long) INITIAL_INFO, &bm_msg, (long) sizeof(struct bm_rm_msg), 
	      (long) i, (long) NODE_PID);
	csend((long) INITIAL_INFO, p4_global->proctable, 
	      (long) sizeof(p4_global->proctable), (long) i, (long) NODE_PID);
#       endif
#       if defined(CM5)
	CMMD_send_noblock(i, INITIAL_INFO, &bm_msg, sizeof(struct bm_rm_msg));
	CMMD_send_noblock(i, INITIAL_INFO, p4_global->proctable, sizeof(p4_global->proctable));
#       endif
	p4_dprintfl(90,"sent initinfo to slave %d of %d\n",i,nslaves);
    }
#   endif

    p4_global->low_cluster_id = 
	p4_local->my_id - p4_global->proctable[p4_local->my_id].slave_idx;
    p4_global->hi_cluster_id = 
	p4_global->low_cluster_id + p4_global->local_slave_count + 1;

    /* 
       sync with local slaves thus insuring that they have the proctable before 
       syncing with remotes (this keeps remotes from interrupting the local 
       processes too early; then re-sync with local slaves (thus permitting them
       to interrupt remotes)
    */

    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());
    /* 
       NEED A SYNC WITH LOCALS THAT DOES A BARRIER WITH PROCS THAT SHARE
       MEMORY AND MP BARRIER WITH OTHER "LOCAL" PROCESSES 
    */
    sync_with_remotes();
    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());


    return (0);
}

int create_bm_processes(pg)
struct p4_procgroup *pg;
{
    struct p4_procgroup_entry *local_pg;
    struct listener_data *l;
    int i, nslaves, end_1, end_2;
    int slave_pid, listener_pid;
    int slave_idx, listener_fd;
    int port, switch_port;
    struct bm_rm_msg bm_msg;

    p4_dprintfl(90,"entering create_bm_processes\n");
    local_pg = &(pg->entries[0]);

    nslaves = local_pg->numslaves_in_group;
#   if !defined(IPSC860)  &&  !defined(CM5)
    if (nslaves > P4_MAX_MSG_QUEUES)
	p4_error("more slaves than msg queues \n", nslaves);
#   endif

/* alloc listener local data since this proc will eventually become listener */
#   ifdef CAN_DO_SOCKET_MSGS
    if (!(p4_global->local_communication_only))
    {
	listener_fd = p4_global->listener_fd;
	listener_info = alloc_listener_info();
	l = listener_info;
	get_pipe(&end_1, &end_2);
    }
#   endif

#   ifdef TCMP
    tcmp_init(NULL,p4_get_my_cluster_id(),shmem_getclunid());
#   endif

#   if defined(IPSC860)  ||  defined(CM5)
    for (i = 1; i <= nslaves; i++)
    {
	p4_dprintfl(90,"doing initial sync with local slave %d\n",i);
#       if defined(IPSC860)
	csend((long) SYNC_MSG, &bm_msg, (long) sizeof(struct bm_rm_msg), 
	      (long) i, (long) NODE_PID);
	crecv(INITIAL_INFO, &bm_msg, (long) sizeof(struct bm_rm_msg));
#       endif
#       if defined(CM5)
	CMMD_send_noblock(i, SYNC_MSG, &bm_msg, sizeof(struct bm_rm_msg));
        CMMD_receive(CMMD_ANY_NODE, INITIAL_INFO, (void *) &bm_msg, sizeof(struct bm_rm_msg));
#       endif
	port = p4_n_to_i(bm_msg.port);
	slave_idx = p4_n_to_i(bm_msg.slave_idx);
	slave_pid = p4_n_to_i(bm_msg.slave_pid);
	switch_port = p4_n_to_i(bm_msg.switch_port);
	/* big master installing local slaves */
	install_in_proctable(0, port, slave_pid, bm_msg.host_name, 
			     slave_idx, P4_MACHINE_TYPE, switch_port);
	p4_global->local_slave_count++;
    }
#   else
    for (slave_idx = 1; slave_idx <= nslaves; slave_idx++)
    {
	p4_dprintfl(20, "creating local slave %d of %d\n",slave_idx,nslaves);
	slave_pid = fork_p4();
	if (slave_pid < 0)
	    p4_error("create_bm_processes fork", slave_pid);
	else
	    if (slave_pid)
		p4_dprintfl(10, "created local slave %d\n", slave_idx);
	if (slave_pid == 0)	/* At this point, we are the slave. */
	{
	    sprintf(whoami, "bm_slave_%d_%d", slave_idx, getpid());

	    p4_free(p4_local);	/* Doesn't work for weird memory model. */
	    p4_local = alloc_local_slave();

#           ifdef CAN_DO_SOCKET_MSGS
	    if (!(p4_global->local_communication_only))
	    {
		p4_local->listener_fd = end_1;
		close(end_2);
		close(listener_fd);
	    }
	    SIGNAL_P4(LISTENER_ATTN_SIGNAL, handle_connection_interrupt);
#           endif

	    /* hang for a valid proctable */
	    p4_lock(&p4_global->slave_lock);
	    p4_unlock(&p4_global->slave_lock);

	    p4_local->my_id = p4_get_my_id_from_proc();
	    setup_conntab();
	    sprintf(whoami, "p%d_%d", p4_local->my_id, getpid());
	    usc_init();

#           ifdef TCMP
            tcmp_init(NULL,p4_get_my_cluster_id(),shmem_getclunid());
#           endif

	    /* 
	       sync with master twice: once to make sure all slaves have 
	       got proctable, and second after the master has synced with the 
	       remote processes 
	    */
	    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());
	    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());

	    p4_dprintfl(20, "local slave starting\n");
	    ALOG_SETUP(p4_local->my_id,ALOG_TRUNCATE);
	    ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");
	    slave();
	    ALOG_LOG(p4_local->my_id,END_USER,0,"");
	    ALOG_OUTPUT;
	    p4_dprintfl(20, "local slave exiting\n");

	    exit(0);
	}

	/* master installing local slaves */
	install_in_proctable(0, p4_global->listener_port, slave_pid,
			     p4_global->my_host_name, 
			     slave_idx, P4_MACHINE_TYPE,
			     p4_global->proctable[0].switch_port);
	p4_global->local_slave_count++;
    }
#   endif

#   if defined(CM5)
    for (i=nslaves+1; i < CMMD_partition_size(); i++)
        CMMD_send_noblock(i, DIE, &bm_msg, sizeof(struct bm_rm_msg));
#   endif


    /* Done creating slaves. Now fork off the listener */

#   if !defined(IPSC860)  &&  !defined(CM5)

#   ifdef CAN_DO_SOCKET_MSGS
    if (!(p4_global->local_communication_only))
    {
	listener_pid = fork_p4();
	if (listener_pid < 0)
	    p4_error("create_bm_processes listener fork", listener_pid);
	if (listener_pid == 0)
	{
	    sprintf(whoami, "bm_list_%d", getpid());
	    /* Inside listener */
	    p4_local = alloc_local_listener();
	    l->listening_fd = listener_fd;
	    l->slave_fd = end_2;
	    close(end_1);
	    listener();
	    exit(0);
	}
    }
#   endif

    /* Else we're still in the big master */
    sprintf(whoami, "p0_%d", getpid());

    /* We need to close the fds from the listener setup */
#   ifdef CAN_DO_SOCKET_MSGS
    if (!(p4_global->local_communication_only))
    {
	p4_local->listener_fd = end_1;
	close(listener_fd);
	close(end_2);
	p4_global->listener_pid = listener_pid;
    }
#   endif

#   endif

    dump_global(80);
    p4_dprintfl(90, "create_bm_processes: exiting\n");
    return (nslaves);
}


P4VOID procgroup_to_proctable(pg)
struct p4_procgroup *pg;
{
    int i, j, ptidx;
    struct p4_procgroup_entry *pe;

    if (strcmp(pg->entries[0].host_name,"local") != 0)
	p4_error("local is not first entry in procgroup ",0);
    strcpy(p4_global->proctable[0].host_name,p4_global->my_host_name);
    get_qualified_hostname(p4_global->proctable[0].host_name);
    p4_global->proctable[0].group_id = 0;
    ptidx = 1;
    for (i=0, pe=pg->entries; i < pg->num_entries; i++, pe++)
    {
	for (j=0; j < pe->numslaves_in_group; j++)
	{
	    if (i == 0)
		strcpy(p4_global->proctable[ptidx].host_name,p4_global->my_host_name);
	    else
		strcpy(p4_global->proctable[ptidx].host_name,pe->host_name);
	    get_qualified_hostname(p4_global->proctable[ptidx].host_name);
	    p4_global->proctable[ptidx].group_id = i;
	    ptidx++;
	}
	p4_global->num_in_proctable = ptidx;
    }
}

P4VOID sync_with_remotes()
{
    struct bm_rm_msg msg;
    int i, fd, node, num_rms, rm[P4_MAXPROCS];

    p4_dprintfl(90, "sync_with_remotes: starting\n");

#   ifdef CAN_DO_SOCKET_MSGS
    p4_get_cluster_masters(&num_rms, rm);
    for (i = 1; i < num_rms; i++)
    {
	node = rm[i];
	fd = p4_local->conntab[node].port;
	net_recv(fd, &msg, sizeof(msg));
	msg.type = p4_n_to_i(msg.type);
	if (msg.type != SYNC_MSG)
	    p4_error("sync_with_remotes: bad type rcvd\n",msg.type);
    }
    for (i = 1; i < num_rms; i++)
    {
	node = rm[i];
	fd = p4_local->conntab[node].port;
	msg.type = p4_i_to_n(SYNC_MSG);
	net_send(fd, &msg, sizeof(msg), FALSE);
    }
#   endif
}

P4VOID send_proc_table()
{
    int slave_idx, ent;
    int fd;
    struct bm_rm_msg msg;
    struct proc_info *pe;

    p4_dprintfl(90, "send_proc_table: starting\n");

#   ifdef CAN_DO_SOCKET_MSGS
    for (slave_idx = 1; slave_idx < p4_global->num_in_proctable; slave_idx++)
    {
	if (p4_global->proctable[slave_idx].slave_idx != 0)
	    continue;

	fd = p4_local->conntab[slave_idx].port;

	p4_dprintfl(90, "sending proctable to slave %d on %d:\n", slave_idx, fd);
	if (fd < 0)
	    p4_error("send_proc_table: rm entry doesn't have valid fd", fd);

	for (ent = 0, pe = p4_global->proctable;
	     ent < p4_global->num_in_proctable; ent++, pe++)
	{
	    msg.type = p4_i_to_n(PROC_TABLE_ENTRY);
	    msg.port = p4_i_to_n(pe->port);
	    msg.unix_id = p4_i_to_n(pe->unix_id);
	    msg.slave_idx = p4_i_to_n(pe->slave_idx);
	    msg.group_id = p4_i_to_n(pe->group_id);
	    strcpy(msg.host_name, pe->host_name);
	    strcpy(msg.machine_type,pe->machine_type);
	    msg.switch_port = p4_i_to_n(pe->switch_port);
	    net_send(fd, &msg, sizeof(msg), FALSE);
	}
	p4_dprintfl(90, "  sending end_of_proc_table\n");
	msg.type = p4_i_to_n(PROC_TABLE_END);
	net_send(fd, &msg, sizeof(msg), FALSE);
    }
#   endif
}
SHAR_EOF
if test -f 'p4_broadcast.c'
then
	echo shar: over-writing existing file "'p4_broadcast.c'"
fi
cat << \SHAR_EOF > 'p4_broadcast.c'
#include "p4.h"
#include "p4_sys.h"

#define MAXVAL(a,b) (((a)>(b)) ? (a) : (b))
#define MINVAL(a,b) (((a)<(b)) ? (a) : (b))
#define ABSVAL(a)   (((a)>=0 ) ? (a) : -(a))

static P4VOID init_p4_brdcst_info();

int p4_broadcastx(type, data, data_len, data_type)
int type;
char *data;
int data_len, data_type;
/*
  Broadcast my data to all other processes.
  Other processes call p4_recv() in the normal fashion, specifying
  the node (if desired) that originated the broadcast.
*/
{
    int status = 0;
    struct p4_msg *tmsg;

    init_p4_brdcst_info();

    /* Build the message with broadcast bit set */

    if (p4_get_my_id())
    {				/* Not node 0 */
	/*
	 * Send data to node 0. Recv internally discards broadcast message
	 * that is from the same process
	 */

	status = send_message(type, p4_get_my_id(), 0, data, data_len,
			      data_type, P4_BROADCAST_MASK, FALSE);
    }
    else
    {
	/* I'm top of the tree ... start broadcast proper */

	status = subtree_broadcast_p4(type, p4_get_my_id(), data, data_len, data_type);
    }

    if (status && !(SOFTERR))
	p4_error("p4_broadcast failed, type=", type);

    return status;
}

int subtree_broadcast_p4(type, from, data, data_len, data_type)
char *data;
int type, from, data_len, data_type;
/*
  Broadcast message to processes in my subtree.

  1) Send to left/right remote cluster masters
  2) Send to left/right local  cluster slaves
*/
{
    int status = 0;
    int size;
    int nodes[4], i;
    struct p4_msg *tmsg;

    init_p4_brdcst_info();

    p4_dprintfl(90, "subtree_broadcast_p4: type=%d, len=%d\n", type, data_len);

    nodes[0] = p4_brdcst_info.left_cluster;
    nodes[1] = p4_brdcst_info.right_cluster;
    nodes[2] = p4_brdcst_info.left_slave;
    nodes[3] = p4_brdcst_info.right_slave;

    for (i = 0; i < 4; i++)
    {
	if (nodes[i] > 0)
	{
	    if (send_message(type, from, nodes[i], data, data_len,
			     data_type, P4_BROADCAST_MASK, FALSE))
	    {
		status = -1;
		break;
	    }
	}
    }

    if (status && !SOFTERR)
	p4_error("subtree_broadcast_p4 failed, type+", tmsg->type);

    p4_dprintfl(90, "subtree_broadcast_p4: exit status=%d\n", status);
    return status;
}

static P4VOID init_p4_brdcst_info()
/*
  Construct tree connections for cluster-master and slave
  processes and insert into global structure
*/
{
#define MAX_MASTERS P4_MAXPROCS
    int me, my_master, node, n_master, indx = (-1);
    int master_list[MAX_MASTERS], previous_id;

    if (p4_brdcst_info.initialized)	/* Only need to do this once */
	return;

    p4_brdcst_info.initialized = 1;
    p4_brdcst_info.up = -1;	/* -1 means no one there */
    p4_brdcst_info.left_cluster = -1;
    p4_brdcst_info.right_cluster = -1;
    p4_brdcst_info.left_slave = -1;
    p4_brdcst_info.right_slave = -1;

    me = p4_get_my_id();

    /* Make list of cluster masters and find my master */
    /* Ideally should probably use p4_get_cluster_masters here */

    n_master = 0;
    previous_id = -1;
    for (node = 0; node < p4_num_total_ids(); node++)
    {
	if (p4_global->proctable[node].group_id != previous_id)
	{
	    master_list[n_master++] = node;
	    previous_id = p4_global->proctable[node].group_id;
	}
	if (node == me)
	    indx = n_master - 1;
    }
    if (indx < 0)
	p4_error("init_p4_brdcst_info: my master indx bad", indx);
    my_master = master_list[indx];

    /*
    printf("me=%d, indx=%d, n_master=%d, my_master=%d, myclusterid=%d, list=[",
	   me, indx, n_master, my_master, p4_get_my_cluster_id()); 
    for (node=0; node<n_master; node++) 
	printf(" %d",master_list[node]); 
    printf(" ]\n");
    */

    if (me == my_master)
    {
	/* If cluster master assign cluster master tree */

	if ((2 * indx + 1) < n_master)
	    p4_brdcst_info.left_cluster = master_list[2 * indx + 1];

	if ((2 * indx + 2) < n_master)
	    p4_brdcst_info.right_cluster = master_list[2 * indx + 2];

	if (me)
	    p4_brdcst_info.up = master_list[(indx - 1) / 2];
    }

    /* Now assign connections with own subtree */
    p4_dprintfl(90, "brdcst_info: numclusids=%d\n", p4_num_cluster_ids());
    node = 2 * p4_get_my_cluster_id() + 1;
    if (node < p4_num_cluster_ids())
	p4_brdcst_info.left_slave = node + my_master;

    node = 2 * p4_get_my_cluster_id() + 2;
    if (node < p4_num_cluster_ids())
	p4_brdcst_info.right_slave = node + my_master;

    if (me != my_master)
	p4_brdcst_info.up = my_master + (p4_get_my_cluster_id() - 1) / 2;

    p4_dprintfl(90, "brdcst_info: me=%d up=%d clusters(%d, %d) slaves(%d,%d)\n",
		me,
		p4_brdcst_info.up,
		p4_brdcst_info.left_cluster,
		p4_brdcst_info.right_cluster,
		p4_brdcst_info.left_slave,
		p4_brdcst_info.right_slave);

    /* Sanity check */

    if (p4_brdcst_info.up != -1)
	if (CHECKNODE(p4_brdcst_info.up))
	    p4_error("init_p4_brdcst_info: up node is invalid", p4_brdcst_info.up);
    if (p4_brdcst_info.left_cluster != -1)
	if (CHECKNODE(p4_brdcst_info.left_cluster))
	    p4_error("init_p4_brdcst_info: left_cluster node is invalid",
		     p4_brdcst_info.left_cluster);
    if (p4_brdcst_info.right_cluster != -1)
	if (CHECKNODE(p4_brdcst_info.right_cluster))
	    p4_error("init_p4_brdcst_info: right_cluster node is invalid",
		     p4_brdcst_info.right_cluster);
    if (p4_brdcst_info.left_slave != -1)
	if (CHECKNODE(p4_brdcst_info.left_slave))
	    p4_error("init_p4_brdcst_info: left_slave node is invalid",
		     p4_brdcst_info.left_slave);
    if (p4_brdcst_info.right_slave != -1)
	if (CHECKNODE(p4_brdcst_info.right_slave))
	    p4_error("init_p4_brdcst_info: right_slave node is invalid",
		     p4_brdcst_info.right_slave);
}


int p4_global_op(type, x, nelem, size, op, data_type)
int type;
char *x;
int nelem;
int size;
int data_type;
P4VOID(*op) ();
/* see userman for more details */
{
    int me = p4_get_my_id();
    int status = 0;
    int zero = 0;
    int msg_len;
    char *msg;

    init_p4_brdcst_info();

    /* Accumulate up broadcast tree ... mess is due to return of soft errors */

    if (!status)
	if (p4_brdcst_info.left_slave > 0)
	{
	    msg = NULL;
	    status = p4_recv(&type, &p4_brdcst_info.left_slave, &msg, &msg_len);
	    if (!status)
	    {
		op(x, msg, msg_len / size);
		p4_msg_free(msg);
	    }
	}
    if (!status)
	if (p4_brdcst_info.right_slave > 0)
	{
	    msg = NULL;
	    status = p4_recv(&type, &p4_brdcst_info.right_slave, &msg, &msg_len);
	    if (!status)
	    {
		op(x, msg, msg_len / size);
		p4_msg_free(msg);
	    }
	}
    if (!status)
	if (p4_brdcst_info.left_cluster > 0)
	{
	    msg = NULL;
	    status = p4_recv(&type, &p4_brdcst_info.left_cluster, &msg, &msg_len);
	    if (!status)
	    {
		op(x, msg, msg_len / size);
		p4_msg_free(msg);
	    }
	}
    if (!status)
	if (p4_brdcst_info.right_cluster > 0)
	{
	    msg = NULL;
	    status = p4_recv(&type, &p4_brdcst_info.right_cluster, &msg, &msg_len);
	    if (!status)
	    {
		op(x, msg, msg_len / size);
		p4_msg_free(msg);
	    }
	}

    if ((!status) && p4_get_my_id())
	status = p4_sendx(type, p4_brdcst_info.up, x, nelem * size, data_type);

    /* Broadcast the result back */

    if (!status)
    {
	if (me == 0)
	    status = p4_broadcastx(type, x, nelem * size, data_type);
	else
	{
	    msg = NULL;
	    status = p4_recv(&type, &zero, &msg, &msg_len);
	    if (!status)
	    {
		bcopy(msg, (char *) x, msg_len);
		p4_msg_free(msg);
	    }
	}
    }

    if (status && !SOFTERR)
	p4_error("p4_global_op failed, type=", type);

    return status;
}

/* Standard operations on doubles */

P4VOID p4_dbl_sum_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
	*a++ += *b++;
}

P4VOID p4_dbl_mult_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
	*a++ *= *b++;
}

P4VOID p4_dbl_max_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
    {
	*a = MAXVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_dbl_min_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
    {
	*a = MINVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_dbl_absmax_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
    {
	*a = MAXVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}

P4VOID p4_dbl_absmin_op(x, y, nelem)
char *x, *y;
int nelem;
{
    double *a = (double *) x;
    double *b = (double *) y;

    while (nelem--)
    {
	*a = MINVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}

/* Standard operations on floats */

P4VOID p4_flt_sum_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
	*a++ += *b++;
}

P4VOID p4_flt_mult_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
	*a++ *= *b++;
}

P4VOID p4_flt_max_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
    {
	*a = MAXVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_flt_min_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
    {
	*a = MINVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_flt_absmax_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
    {
	*a = MAXVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}

P4VOID p4_flt_absmin_op(x, y, nelem)
char *x, *y;
int nelem;
{
    float *a = (float *) x;
    float *b = (float *) y;

    while (nelem--)
    {
	*a = MINVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}


/* Standard operations on integers */

P4VOID p4_int_sum_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
	*a++ += *b++;
}

P4VOID p4_int_mult_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
	*a++ *= *b++;
}

P4VOID p4_int_max_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
    {
	*a = MAXVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_int_min_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
    {
	*a = MINVAL(*a, *b);
	a++;
	b++;
    }
}

P4VOID p4_int_absmax_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
    {
	*a = MAXVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}

P4VOID p4_int_absmin_op(x, y, nelem)
char *x, *y;
int nelem;
{
    int *a = (int *) x;
    int *b = (int *) y;

    while (nelem--)
    {
	*a = MINVAL(ABSVAL(*a), ABSVAL(*b));
	a++;
	b++;
    }
}
SHAR_EOF
if test -f 'p4_cmain.c'
then
	echo shar: over-writing existing file "'p4_cmain.c'"
fi
cat << \SHAR_EOF > 'p4_cmain.c'
#include "p4.h"

main(argc, argv)
int argc;
char **argv;
{
    p4_initenv(&argc, argv);
    slave();
    p4_wait_for_end();
}
SHAR_EOF
if test -f 'p4_debug.c'
then
	echo shar: over-writing existing file "'p4_debug.c'"
fi
cat << \SHAR_EOF > 'p4_debug.c'
#include <stdio.h>
#include <varargs.h>
#include "p4.h"
#include "p4_sys.h"

#if defined(p4_dprintfl)
#undef p4_dprintfl
#endif

int p4_get_dbg_level()
{
    return(debug_level);
}

P4VOID p4_set_dbg_level(level)
int level;
{
    debug_level = level;
}

#ifdef DELTA

P4VOID p4_dprintf(fmt, a, b, c, d, e, f, g, h, i)
{
    printf("%s: ",whoami);
    printf(fmt,a,b,c,d,e,f,g,h,i);
}

#else

P4VOID p4_dprintf(fmt, va_alist)
char *fmt;
va_dcl
{
    va_list ap;

    printf("%s: ", whoami);
    va_start(ap);
#ifdef VPRINTF
    vprintf(fmt, ap);
#else
    _doprnt(fmt, ap, stdout);
#endif
    va_end(ap);
    fflush(stdout);
}

#endif

#ifdef DELTA

P4VOID p4_dprintfl(level, fmt, a, b, c, d, e, f, g, h, i)
{
    if (level > debug_level)
        return;
    printf("%s: ",whoami);
    printf(fmt,a,b,c,d,e,f,g,h,i);
}

#else

P4VOID p4_dprintfl(level, fmt, va_alist)
int level;
char *fmt;
va_dcl
{
    va_list ap;

    if (level > debug_level)
	return;
    printf("%d: %s: ", level, whoami);
    va_start(ap);
#ifdef VPRINTF
    vprintf(fmt, ap);
#else
    _doprnt(fmt, ap, stdout);
#endif
    va_end(ap);
    fflush(stdout);
}

#endif 

P4VOID dump_global(level)
int level;
{
    int i;
    struct p4_global_data *g = p4_global;
    struct proc_info *p;

    if (level > debug_level)
	return;

    p4_dprintf("Dumping global data for process %d at %x\n", getpid(), g);

    for (i = 0, p = g->proctable; i < g->num_in_proctable; i++, p++)
    {
	p4_dprintf(" proctable entry %d: unix_id = %d host = %s\n",
		   i, p->unix_id, p->host_name);
	p4_dprintf("   port=%d group_id=%d switch_port=%d\n",
		   p->port, p->group_id, p->switch_port);
    }

    p4_dprintf("    listener_pid     = %d\n", g->listener_pid);
    p4_dprintf("    listener_port    = %d\n", g->listener_port);
    p4_dprintf("    local_slave_count= %d\n", g->local_slave_count);
    p4_dprintf("    my_host_name     = %s\n", g->my_host_name);
    p4_dprintf("    num_in_proctable = %d\n", g->num_in_proctable);
}

P4VOID dump_local(level)
int level;
{
    struct local_data *l = p4_local;
    int i;

    if (level > debug_level)
	return;

    p4_dprintf("Dumping local data for process %d at %x\n", getpid(), l);

    for (i = 0; i < p4_global->num_in_proctable; i++)
	p4_dprintf("     %d: conntab[%d]  type:%s    port %d\n", getpid(), i,
		   print_conn_type(p4_local->conntab[i].type),
		   p4_local->conntab[i].port);

    p4_dprintf("    listener_fd = %d\n", l->listener_fd);
    p4_dprintf("    my_id       = %d\n", l->my_id);
    p4_dprintf("    am_bm       = %d\n", l->am_bm);
}

char *print_conn_type(conn_type)
int conn_type;
{
    static char val[20];

    switch (conn_type)
    {
      case CONN_ME:
	return ("CONN_ME");
      case CONN_REMOTE_SWITCH:
	return ("CONN_REMOTE_SWITCH");
      case CONN_REMOTE_NON_EST:
	return ("CONN_REMOTE_NON_EST");
      case CONN_REMOTE_EST:
	return ("CONN_REMOTE_EST");
      case CONN_SHMEM:
	return ("CONN_SHMEM");
      case CONN_CUBE:
	return ("CONN_CUBE");
      case CONN_REMOTE_DYING:
	return ("CONN_REMOTE_DYING");
      default:
	sprintf(val, "invalid: %d  ", conn_type);
	return (val);
    }
}


P4VOID dump_listener(level)
int level;
{
    struct listener_data *l = listener_info;

    if (level > debug_level)
	return;

    p4_dprintf("Dumping listener data for process %d at %x\n", getpid(), l);
    p4_dprintf("    listening_fd = %d\n", l->listening_fd);
}

P4VOID dump_procgroup(procgroup, level)
struct p4_procgroup *procgroup;
int level;
{
    struct p4_procgroup_entry *pe;
    int i;

    if (level > debug_level)
	return;

    p4_dprintf("Procgroup:\n");
    for (pe = procgroup->entries, i = 0;
	 i < procgroup->num_entries;
	 pe++, i++)
	p4_dprintf("    entry %d: %s %d %s %s \n",
		   i,
		   pe->host_name,
		   pe->numslaves_in_group,
		   pe->slave_full_pathname,
		   pe->username);
}

P4VOID dump_tmsg(tmsg)
struct p4_msg *tmsg;
{
    p4_dprintf("type=%d, to=%d, from=%d, len=%d, ack_req=%x, msg=%s\n",
	       tmsg->type, tmsg->to, tmsg->from, tmsg->len, tmsg->ack_req,
	       &(tmsg->msg));
}

P4VOID dump_conntab(level)
int level;
{
    int i;

    if (level > debug_level)
	return;

    for (i = 0; i < p4_global->num_in_proctable; i++)
    {
	p4_dprintf("   %d: conntab[%d] type=%s port=%d switch_port=%d\n",
		   getpid(), i,
		   print_conn_type(p4_local->conntab[i].type),
		   p4_local->conntab[i].port,
		   p4_local->conntab[i].switch_port);
    }
}
SHAR_EOF
if test -f 'p4_defs.h'
then
	echo shar: over-writing existing file "'p4_defs.h'"
fi
cat << \SHAR_EOF > 'p4_defs.h'
struct proc_info {
    int port;
    int switch_port;
    int unix_id;
    int slave_idx;
    int group_id;
    P4BOOL am_rm;
    char host_name[HOSTNAME_LEN];
    char machine_type[16];
};

#define NUMAVAILS 8

struct p4_avail_buff {
    int size;			/* size of message portion */
    struct p4_msg *buff;
};

struct p4_global_data {
#   ifdef SYSV_IPC
    int sysv_num_semids;
    int sysv_semid[P4_MAX_SYSV_SEMIDS];
    int sysv_next_lock;
#   endif
    struct proc_info proctable[P4_MAXPROCS];
    int listener_pid;
    int listener_port;
    P4BOOL local_communication_only;
    int local_slave_count;
    int n_forked_pids;
    char my_host_name[HOSTNAME_LEN];
    struct p4_avail_buff avail_buffs[NUMAVAILS];
    p4_lock_t avail_buffs_lock;
    struct p4_queued_msg *avail_quel;
    p4_lock_t avail_quel_lock;
    struct p4_msg_queue shmem_msg_queues[P4_MAX_MSG_QUEUES];
    int num_in_proctable;
    int num_installed;
    p4_lock_t slave_lock;
    int dest_id[P4_MAXPROCS];
    int listener_fd;
    int max_connections;
    int cube_msgs_out;    /* i860 msgs not yet msgwait'ed on */
    long reference_time;  /* used in p4_initenv and p4_clock */
    int hi_cluster_id;
    int low_cluster_id;
    P4VOID *cluster_shmem;
    p4_barrier_monitor_t cluster_barrier;
    char application_id[12];
} *p4_global;


struct connection {
    int type;
    int port;
    int switch_port;
    P4BOOL same_data_rep;
};

struct local_data {		/* local to each process */
    int listener_fd;
    int my_id;
    int local_commtype;		/* cube or shmem messages */
    struct p4_msg_queue *queued_messages;
    P4BOOL am_bm;
    struct connection *conntab;	/* pointer to array of connections */
    struct p4_procgroup *procgroup;
    int soft_errors;            /* false if errors cause termination */
    char *xdr_buff;
    XDR xdr_enc;
    XDR xdr_dec;
} *p4_local;

struct listener_data {
    int listening_fd;
    int slave_fd;
} *listener_info;


/* this struct is similar to a p4_net_msg_hdr;  note that the sum of
   the sizes of the items up to the *msg is equal to some number of 
   double wrods, which is important on machines like bfly2 if you 
   receive doubles into the msg area.
*/
struct p4_msg {
    struct p4_msg *link;
    int type;                
    int to;
    int from;
    int ack_req;
    int len;
    int msg_id;		        /* for i860 messages */
    int data_type;		/* for use by xdr */
    char *msg;	/* variable length array of characters */
};

struct p4_net_msg_hdr {
    int msg_type:32;
    int to:32;
    int from:32;
    int ack_req:32;
    int msg_len:32;
    int msg_id:32;		/* for i860 messages */
    int data_type:32;		/* for use by xdr */
    int pad:32;                 /* pad field to word boundary */
};

struct net_initial_handshake {
   int pid:32;
   int pad:32;
};

struct p4_queued_msg {
    struct p4_msg *qmsg;
    struct p4_queued_msg *next;
};


/* Messages between a listener and any other non-listener */

#define DIE   1
#define SLAVE_DYING   2     /* Unused.  Check for whole data struct. */
#define CONNECTION_REQUEST   3
#define IGNORE_THIS   4

struct slave_listener_msg {
    int type:32;
    int from:32;
    int to:32;
    int to_pid:32;
    int lport:32;
    int pad:32;
};

/* Messages between the bm and a rm at startup */

#define INITIAL_INFO            11
#define REMOTE_LISTENER_INFO    12
#define REMOTE_SLAVE_INFO       13
#define REMOTE_MASTER_INFO      14
#define REMOTE_SLAVE_INFO_END   15
#define PROC_TABLE_ENTRY        16
#define PROC_TABLE_END          17
#define SYNC_MSG                18

struct bm_rm_msg {
    int type:32;

    /* for INITIAL_INFO */
    int numslaves:32;
    int numinproctab:32;
    int memsize:32;
    int rm_num:32;
    int debug_level:32;
    int logging_flag:32;

    /* for REMOTE_LISTENER_INFO */
    int port:32;

    /* for REMOTE_SLAVE_INFO and REMOTE_MASTER_INFO */
    int slave_idx:32;
    int slave_pid:32;
    int am_rm:32;

    /* for PROC_TABLE_ENTRY */
    int unix_id:32;
    int group_id:32;
    int switch_port:32;
    /* int pad:32;  to keep number of 32 bit quantities even */
    char host_name[HOSTNAME_LEN];

    /* also for INITIAL INFO */
    char pgm[128];
    char version[8];
    char outfile[128];
    char application_id[16];
    char machine_type[16];
};

#define P4_ACK_REQ_MASK   1     /* Masks define bits set for requests */
#define P4_ACK_REPLY_MASK 2
#define P4_BROADCAST_MASK 4

struct p4_brdcst_info_struct {
/*
  This structure is initialized by init_p4_brdcst_info() which
  is automatically called by every global operation
*/
  int initialized;             /* True if structure is initialized */
  int up;                      /* Process above me in tree         */
  int left_cluster;            /* Id of left child cluster master  */
  int right_cluster;           /* Id of right child cluster master */
  int left_slave;              /* Id of left child slave           */
  int right_slave;             /* Id of right child slave          */
} p4_brdcst_info;
SHAR_EOF
if test -f 'p4_error.c'
then
	echo shar: over-writing existing file "'p4_error.c'"
fi
cat << \SHAR_EOF > 'p4_error.c'
#include "p4.h"
#include "p4_sys.h"

extern P4VOID exit();

static int interrupt_caught = 0; /* True if an interrupt was caught */

#if defined(ENCORE) || defined(SYMMETRY) || defined(TITAN) || \
    defined(SGI)    || defined(GP_1000)  || defined(TC_2000)
#define P4_HANDLER_TYPE int
#else
#define P4_HANDLER_TYPE P4VOID
#endif

static P4_HANDLER_TYPE (*prev_sigint_handler) () = NULL;
static P4_HANDLER_TYPE (*prev_sigsegv_handler) () = NULL;
static P4_HANDLER_TYPE (*prev_sigbus_handler) () = NULL;
static P4_HANDLER_TYPE (*prev_sigfpe_handler) () = NULL;
static P4_HANDLER_TYPE (*prev_err_handler) () = NULL;
static int err_sig, err_code;
static struct sigcontext *err_scp;
static char *err_addr;


int p4_soft_errors(onoff)
int onoff;
{
    int old;

    if (!p4_local)
	p4_error("p4_soft_errors: p4_local must be allocated first", 0);

    old = p4_local->soft_errors;
    p4_local->soft_errors = onoff;
    return old;
}

P4VOID p4_error(string, value)
char *string;
int value;
{
    SIGNAL_P4(SIGINT,SIG_IGN);
    fflush(stdout);
    printf("%s:  p4_error: %s: %d\n",whoami,string,value);
    if (value < 0)
        perror("    p4_error: latest msg from perror");
    fflush(stdout);

    /* Send interrupt to all known processes */
    zap_p4_processes();

    /* shutdown(sock,2), close(sock) all sockets */
#   ifdef CAN_DO_SOCKET_MSGS
    shutdown_p4_socks();
#   endif

#   ifdef SYSV_IPC
    remove_sysv_ipc();
#   endif

    if (interrupt_caught && value != SIGINT)
    {
	switch (value)
	{
	  case SIGSEGV:
	    prev_err_handler = prev_sigsegv_handler;
	    break;
	  case SIGBUS:
	    prev_err_handler = prev_sigbus_handler;
	    break;
	  case SIGFPE:
	    prev_err_handler = prev_sigfpe_handler;
	    break;
	  default:
	    printf("p4_error: unidentified err handler\n");
	    prev_err_handler = NULL;
	    break;
	}
	if (prev_err_handler == (P4_HANDLER_TYPE (*) ()) NULL)
	{
	    /* return to default handling of the interrupt by the OS */
	    SIGNAL_P4(value,SIG_DFL); 
#           if defined(NEXT)  ||  defined(KSR)
            kill(getpid(),value);
#           endif
	    return;
	}
	else
	{
	    (*prev_err_handler) (err_sig, err_code, err_scp, err_addr);
	}
    }
    else
    {
	exit(1);
    }
}

static P4_HANDLER_TYPE sig_err_handler(sig, code, scp, addr)
int sig, code;
struct sigcontext *scp;
char *addr;
{
    interrupt_caught = 1;
    err_sig = sig;
    err_code = code;
    err_scp = scp;
    err_addr = addr;
    if (sig == 11)
	p4_error("interrupt SIGSEGV", sig);
    else if (sig == 10)
	p4_error("interrupt SIGBUS", sig);
    else if (sig == 8)
	p4_error("interrupt SIGFPE", sig);
    else if (sig == 2)
	p4_error("interrupt SIGINT", sig);
    else
	p4_error("interrupt SIGx", sig);
}


/*
  Trap signals so that we can propagate error conditions and tidy up 
  shared system resources in a manner not possible just by killing procs
*/
P4VOID trap_sig_errs()
{
    P4_HANDLER_TYPE (*rc) ();

    rc = SIGNAL_P4(SIGINT, sig_err_handler);
    if (rc == (P4_HANDLER_TYPE (*) ()) -1)
	p4_error("trap_sig_errs: SIGNAL_P4 failed", SIGINT);
    if (((int) rc > 1)  &&  (rc != sig_err_handler))
	prev_sigint_handler = rc;

/* we can not handle sigsegv on symmetry and balance because they use 
 * it for shmem stuff 
*/
#ifdef CAN_HANDLE_SIGSEGV
    rc = SIGNAL_P4(SIGSEGV, sig_err_handler);
    if ((int) rc == -1)
	p4_error("trap_sig_errs: SIGNAL_P4 failed", SIGSEGV);
    if (((int) rc > 1)  &&  (rc != sig_err_handler))
	prev_sigsegv_handler = rc;
#endif

    rc = SIGNAL_P4(SIGBUS, sig_err_handler);
    if ((int) rc == -1)
	p4_error("trap_sig_errs: SIGNAL_P4 failed", SIGBUS);
    if (((int) rc > 1)  &&  (rc != sig_err_handler))
	prev_sigbus_handler = rc;
    rc = SIGNAL_P4(SIGFPE, sig_err_handler);
    if ((int) rc == -1)
	p4_error("trap_sig_errs: SIGNAL_P4 failed", SIGFPE);
    if (((int) rc > 1)  &&  (rc != sig_err_handler))
	prev_sigfpe_handler = rc;
}
SHAR_EOF
if test -f 'p4_funcs.h'
then
	echo shar: over-writing existing file "'p4_funcs.h'"
fi
cat << \SHAR_EOF > 'p4_funcs.h'
char *p4_shmalloc();
P4BOOL p4_am_i_cluster_master();
int p4_askfor();
int p4_askfor_init();
P4VOID p4_barrier();
int p4_barrier_init();
int p4_broadcastx();
int p4_clock();
int p4_create();
int p4_create_procgroup();
int p4_startup();
P4VOID p4_dprintf();
P4VOID p4_error();
P4VOID p4_global_barrier();
P4VOID p4_get_cluster_masters();
P4VOID p4_get_cluster_ids();
int p4_get_my_cluster_id();
int p4_get_my_id();
int p4_get_my_id_from_proc();
int p4_getsub_init();
P4VOID p4_getsubs();
int p4_global_op();
int p4_initenv();
P4VOID p4_int_absmax_op();
P4VOID p4_int_absmin_op();
P4VOID p4_int_max_op();
P4VOID p4_int_min_op();
P4VOID p4_int_mult_op();
P4VOID p4_int_sum_op();
P4VOID p4_dbl_absmax_op();
P4VOID p4_dbl_absmin_op();
P4VOID p4_dbl_max_op();
P4VOID p4_dbl_min_op();
P4VOID p4_dbl_mult_op();
P4VOID p4_dbl_sum_op();
P4VOID p4_flt_sum_op();
P4VOID p4_flt_absmax_op();
P4VOID p4_flt_absmin_op();
P4VOID p4_flt_max_op();
P4VOID p4_flt_min_op();
P4VOID p4_flt_mult_op();
P4VOID p4_flt_sum_op();
P4VOID p4_mcontinue();
P4VOID p4_mdelay();
P4VOID p4_menter();
P4BOOL p4_messages_available();
P4VOID p4_mexit();
int p4_moninit();
P4VOID p4_msg_free();
char *p4_msg_alloc();
int p4_num_cluster_ids();
int p4_num_total_ids();
int p4_num_total_slaves();
P4VOID p4_probend();
P4VOID p4_progend();
int p4_recv();
int p4_get_dbg_level();
P4VOID p4_set_dbg_level();
P4VOID p4_shfree();
int p4_soft_errors();
P4VOID p4_update();
char *p4_version();
char *p4_machine_type();
int p4_wait_for_end();
P4VOID p4_print_avail_buffs();

#ifdef P4_DPRINTFL
P4VOID p4_dprintfl();
#endif
SHAR_EOF
if test -f 'p4_globals.c'
then
	echo shar: over-writing existing file "'p4_globals.c'"
fi
cat << \SHAR_EOF > 'p4_globals.c'
#define GLOBAL
#include "p4.h"
#include "p4_sys.h"
SHAR_EOF
if test -f 'p4_globals.h'
then
	echo shar: over-writing existing file "'p4_globals.h'"
fi
cat << \SHAR_EOF > 'p4_globals.h'
#ifdef GLOBAL
#define PUBLIC
#else
#define PUBLIC extern
#endif
    
/* Debugging information */

PUBLIC char procgroup_file[100];
PUBLIC char bm_outfile[100];
PUBLIC char rm_outfile_head[100];
PUBLIC char whoami[100];
PUBLIC int  debug_level, remote_debug_level;
PUBLIC int  logging_flag;

/* Other global data */

PUBLIC char local_domain[100];
PUBLIC int  globmemsize;
PUBLIC int  sserver_port;

#ifdef SYSV_IPC
PUBLIC int sysv_num_shmids;
PUBLIC int sysv_shmid[P4_MAX_SYSV_SHMIDS];
PUBLIC int sysv_semid0;
#endif
SHAR_EOF
if test -f 'p4_macros.h'
then
	echo shar: over-writing existing file "'p4_macros.h'"
fi
cat << \SHAR_EOF > 'p4_macros.h'
#define get_proc_info(id) (&(p4_global->proctable[id]))

#ifndef FD_ZERO
#    define FD_ZERO(ptr) ((ptr)->fds_bits[0]) = 0;
#    define FD_SET(fd,ptr) ((ptr)->fds_bits[0]) |= 1 << fd;
#    define FD_ISSET(fd,ptr) (((ptr)->fds_bits[0]) & (1 << fd))
#endif

#ifdef P4SYSV
/*
 * Select uses bit masks of file descriptors in longs.
 * These macros manipulate such bit fields (the filesystem macros use chars).
 * FD_SETSIZE may be defined by the user.
 */

#ifndef FD_SETSIZE
#define FD_SETSIZE      256
#endif

#ifndef howmany
#define howmany(x, y)   (((x)+((y)-1))/(y))
#endif

#ifndef NFDBITS
typedef long    fd_mask;
#define NFDBITS (sizeof(fd_mask) * 8)           /* bits per mask */
typedef struct fd_set {
	fd_mask fds_bits[howmany(FD_SETSIZE, NFDBITS)];
} fd_set;
#endif

#endif

#if defined(P4SYSV)
#    ifndef index
#        define index(S,C)        strchr((S),(C))
#    endif
#    ifndef rindex
#        define rindex(S,C)       strrchr((S),(C))
#    endif
#    define bcopy(x,y,len)    memcpy((y),(x),(len))
#    define bcmp(x,y,len)     memcmp((y),(x),(len))
#    define bzero(x,len)      memset((x),0,(len))
#endif

#define SOFTERR (p4_local->soft_errors)
SHAR_EOF
if test -f 'p4_mon.c'
then
	echo shar: over-writing existing file "'p4_mon.c'"
fi
cat << \SHAR_EOF > 'p4_mon.c'
#include "p4.h"
#include "p4_sys.h"

int p4_moninit(m, i)
p4_monitor_t *m;
int i;
{
    int j;
    struct p4_mon_queue *q;

    p4_lock_init(&(m->mon_lock));

    if (i)
    {
	m->qs = (struct p4_mon_queue *) p4_shmalloc(sizeof(struct p4_mon_queue) * i);
	if (m->qs == NULL)
	{
	    p4_dprintf("OOPS! p4_moninit: p4_shmalloc failed ***\n");
	    return (-1);
	}
	for (j = 0; j < i; j++)
	{
	    q = m->qs + j;
	    q->count = 0;
	    p4_lock_init(&(q->delay_lock));
	    p4_lock(&(q->delay_lock));
	}
    }
    else
	m->qs = NULL;
    return (0);
}

P4VOID p4_menter(m)
p4_monitor_t *m;
{
    ALOG_LOG(p4_local->my_id,REQUEST_MONITOR_ENTRY,m,"");
    p4_lock(&(m->mon_lock));
    ALOG_LOG(p4_local->my_id,ENTER_MONITOR,m,"");
}

P4VOID p4_mexit(m)
p4_monitor_t *m;
{
    ALOG_LOG(p4_local->my_id,OPEN_DOOR,m,"");
    ALOG_LOG(p4_local->my_id,EXIT_MONITOR,m,"");
    p4_unlock(&(m->mon_lock));
}

P4VOID p4_mdelay(m, i)
p4_monitor_t *m;
int i;
{
    struct p4_mon_queue *q;

    q = m->qs + i;
    q->count++;
    ALOG_LOG(p4_local->my_id,ENTER_DELAY_QUEUE,m,"");
    ALOG_LOG(p4_local->my_id,OPEN_DOOR,m,"");
    p4_unlock(&(m->mon_lock));
    p4_lock(&(q->delay_lock));
    ALOG_LOG(p4_local->my_id,EXIT_DELAY_QUEUE,m,"");
}

P4VOID p4_mcontinue(m, i)
p4_monitor_t *m;
int i;
{
    struct p4_mon_queue *q;

    q = m->qs + i;
    if (q->count)
    {
	q->count--;
	ALOG_LOG(p4_local->my_id,SECRET_EXIT_MONITOR,m,"");
	p4_unlock(&(q->delay_lock));
    }
    else
    {
	ALOG_LOG(p4_local->my_id,OPEN_DOOR,m,"");
	ALOG_LOG(p4_local->my_id,EXIT_MONITOR,m,"");
	p4_unlock(&(m->mon_lock));
    }
}

int num_in_mon_queue(m, i)
int i;
p4_monitor_t *m;
{
    struct p4_mon_queue *q;

    q = m->qs + i;
    return (q->count);
}


/* ------------------  getsub monitor -------------------- */

int p4_getsub_init(gs)
p4_getsub_monitor_t *gs;
{

    gs->sub = 0;
    return (p4_moninit(&(gs->m), 1));
}

P4VOID p4_getsubs(gs, s, max, nprocs, stride)
p4_getsub_monitor_t *gs;
int *s, max, nprocs, stride;
{
    p4_menter(&(gs->m));
    if (gs->sub <= max)
    {
	*s = gs->sub;
	gs->sub += stride;
	p4_mexit(&(gs->m));
    }
    else
    {
	*s = -1;
	if (num_in_mon_queue(&(gs->m), 0) < nprocs - 1)
	    p4_mdelay(&(gs->m), 0);
	else
	    gs->sub = 0;
	p4_mcontinue(&(gs->m), 0);
    }
}


/* ------------------  barrier monitor -------------------- */

int p4_barrier_init(b)
p4_barrier_monitor_t *b;
{

    return (p4_moninit(&(b->m), 1));
}

P4VOID p4_barrier(b, nprocs)
p4_barrier_monitor_t *b;
int nprocs;
{
    p4_menter(&(b->m));
    if (num_in_mon_queue(&(b->m), 0) < nprocs - 1)
	p4_mdelay(&(b->m), 0);
    p4_mcontinue(&(b->m), 0);
}


/* ------------------  askfor monitor -------------------- */

int p4_askfor_init(af)
p4_askfor_monitor_t *af;
{

    af->pgdone = 0;
    af->pbdone = 0;
    /* alog assumes only one askfor per program */
    ALOG_LOG(p4_local->my_id,PBDONE,0,"");
    ALOG_LOG(p4_local->my_id,PGDONE,0,"");
    ALOG_LOG(p4_local->my_id,UPDATE_NUM_SUBPROBS,0,"");
    return (p4_moninit(&(af->m), 1));
}

int p4_askfor(af, nprocs, getprob_fxn, problem, reset_fxn)
p4_askfor_monitor_t *af;
int nprocs;
int (*getprob_fxn) ();
P4VOID *problem;
P4VOID(*reset_fxn) ();
{
    int rc;

    p4_menter(&(af->m));
    if (!(af->pgdone) && af->pbdone)
    {
	if (num_in_mon_queue(&(af->m), 0) < nprocs - 1)
	{
	    p4_mdelay(&(af->m), 0);
	}
    }
    else
    {
	while (!(af->pgdone) && !(af->pbdone))
	{
	    if ((rc = (*getprob_fxn) (problem)) == 0)
	    {
		p4_mcontinue(&(af->m), 0);
		return (rc);
	    }
	    else
	    {
		if (num_in_mon_queue(&(af->m), 0) == nprocs - 1)
		{
		    af->pbdone = 1;
		    ALOG_LOG(p4_local->my_id,PBDONE,1,"");
		}
		else
		{
		    p4_mdelay(&(af->m), 0);
		}
	    }
	}
    }
    if (af->pgdone)
    {
	rc = (-1);
	p4_mcontinue(&(af->m), 0);
    }
    else
    {
	rc = af->pbdone;
	if (num_in_mon_queue(&(af->m), 0) == 0)
	{
	    (*reset_fxn) ();
	    af->pbdone = 0;
	}
	p4_mcontinue(&(af->m), 0);
    }
    return (rc);
}

P4VOID p4_update(af, putprob_fxn, problem)
p4_askfor_monitor_t *af;
int (*putprob_fxn) ();
P4VOID *problem;
{
    p4_menter(&(af->m));
    if (putprob_fxn(problem))
	p4_mcontinue(&(af->m), 0);
    else
	p4_mexit(&(af->m));
}

P4VOID p4_probend(af, code)
p4_askfor_monitor_t *af;
int code;
{
    p4_menter(&(af->m));
    af->pbdone = code;
    ALOG_LOG(p4_local->my_id,PBDONE,code,"");
    p4_mexit(&(af->m));
}

P4VOID p4_progend(af)
p4_askfor_monitor_t *af;
{
    p4_menter(&(af->m));
    af->pgdone = 1;
    ALOG_LOG(p4_local->my_id,PGDONE,1,"");
    p4_mcontinue(&(af->m), 0);
}

int p4_create(fxn)
int (*fxn) ();
{
    int rc;

    p4_dprintfl(20,"creating local slave via fork\n");
    if ((rc = fork_p4()) == 0)
    {
	/* slave process */
	(*fxn) ();
	exit(0);
    }
    /* else master process */
    p4_dprintfl(10,"created local slave via fork\n");
    return (rc);
}
SHAR_EOF
if test -f 'p4_mon.h'
then
	echo shar: over-writing existing file "'p4_mon.h'"
fi
cat << \SHAR_EOF > 'p4_mon.h'

typedef MD_lock_t p4_lock_t;

#define p4_lock_init(l)  MD_lock_init(l)
#define p4_lock(l)       MD_lock(l)
#define p4_unlock(l)     MD_unlock(l)

struct p4_monitor {
    p4_lock_t mon_lock;
    struct p4_mon_queue *qs;
};

typedef struct p4_monitor p4_monitor_t;


struct p4_mon_queue {
    int count;
    p4_lock_t delay_lock;
};


struct p4_getsub_monitor {
    struct p4_monitor m;
    int sub;
};

typedef struct p4_getsub_monitor p4_getsub_monitor_t;

#define p4_getsub(gs,s,max,nprocs) p4_getsubs(gs,s,max,nprocs,1)

struct p4_barrier_monitor {
    struct p4_monitor m;
};

typedef struct p4_barrier_monitor p4_barrier_monitor_t;

struct p4_askfor_monitor {
    struct p4_monitor m;
    int pgdone;
    int pbdone;
};

typedef struct p4_askfor_monitor p4_askfor_monitor_t;

SHAR_EOF
if test -f 'p4_procgroup.c'
then
	echo shar: over-writing existing file "'p4_procgroup.c'"
fi
cat << \SHAR_EOF > 'p4_procgroup.c'
#include "p4.h"
#include "p4_sys.h"

#ifdef TCMP
extern struct tc_globmem *tcglob;
#endif

struct p4_procgroup *p4_alloc_procgroup()
{
    struct p4_procgroup *pg;

    if (!(pg = (struct p4_procgroup *) p4_malloc(sizeof(struct p4_procgroup))))
	p4_error("p4_alloc_procgroup: p4_malloc failed",
		 sizeof(struct p4_procgroup));

    p4_dprintfl(90, "p4_alloc_procgroup: allocing %d bytes\n",
		sizeof(struct p4_procgroup));

    pg->num_entries = 0;
    return (pg);
}

struct p4_procgroup *read_procgroup()
{
    FILE *fp;
    char buf[1024], *s;
    struct p4_procgroup_entry *pe;
    int i, group_id, pt_index, n;
    struct p4_procgroup *pg;
    struct passwd *pwent;
    char *logname; 


    p4_dprintfl(90,"entering read_procgroup\n");
    pg = p4_alloc_procgroup();

#   if defined(CM5)
    logname = '\0';
#   else
    logname = (char *) getlogin();
#   endif

    if ((fp = fopen(procgroup_file, "r")) == NULL)
	p4_error("open error on procgroup file",NULL);

    pe = pg->entries;

    while (fgets(buf, sizeof(buf), fp) != NULL)
    {
	for (s = buf; isspace(*s); s++)
	    ;

	if (*s == '#' || *s == '\0')	/* Ignore comments & blanks */
	    continue;

	n = sscanf(buf, "%s %d %s %s",
		   pe->host_name,
		   &pe->numslaves_in_group,
		   pe->slave_full_pathname,
		   pe->username);

	if (n == 3)
	{
	    if (logname != NULL && logname[0] != '\0')
		strcpy(pe->username, logname);
	    else
	    {
#               if defined(CM5)
                strcpy(pe->username, "cm5-user");
#               else
		if ((pwent = getpwuid(getuid())) == NULL)
		    p4_error("create_procgroup: getpwuid failed", 0);
		strcpy(pe->username, pwent->pw_name);
#               endif
	    }
	}
	pe++;
	pg->num_entries++;
	if (pg->num_entries > P4_MAX_PROCGROUP_ENTRIES)
	    p4_error("read procgroup: exceeded max # of procs",
		     P4_MAX_PROCGROUP_ENTRIES);
    }

    dump_procgroup(pg,50);
    return (pg);
}				/* read_procgroup */


int install_in_proctable(group_id,port,unix_id,host_name,
			 slv_idx,machine_type,switch_port)
int group_id;
int port;
int unix_id;
char host_name[64];
int slv_idx;
char machine_type[];
int switch_port;
{
    struct p4_global_data *g;
    struct proc_info *pi;

    g = p4_global;
    pi = &g->proctable[g->num_installed];
    pi->group_id = group_id;
    pi->port = port;
    pi->unix_id = unix_id;
    strcpy(pi->host_name, host_name);
    strcpy(pi->machine_type,machine_type);
    pi->slave_idx = slv_idx;
    pi->switch_port = switch_port;
    g->num_installed++;
    p4_dprintfl(50, "installed in proctable num=%d port=%d host=%s unix=%d slav=%d grp=%d swport=%d\n",
		g->num_installed, port, host_name, unix_id, slv_idx, pi->group_id,pi->switch_port);
    return (g->num_installed - 1);
}

SHAR_EOF
if test -f 'p4_secure.c'
then
	echo shar: over-writing existing file "'p4_secure.c'"
fi
cat << \SHAR_EOF > 'p4_secure.c'
#include "p4.h"
#include "p4_sys.h"

#if defined(SYMMETRY) || defined(SUN) || defined(DEC5000) || defined(SGI)

/**********************************
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>
#include <pwd.h>
#include <stdio.h>
**********************************/

/* #define DEBUG */

char *start_prog_error;

extern int errno;
extern char *sys_errlist[];

static int connect_to_server();
static void send_string();
static void recv_string();

int start_slave(host, username, prog, port, am_slave, pw_hook)
char *host, *username, *prog, *am_slave;
int port;
char *(*pw_hook) ();
{
    int n, conn;
    struct passwd *pw;
    char port_string[250];
    char *pw_string;
    static char buf[250];
    char *local_username;
    char myhost[256];
    int new_port, new_fd, stdout_fd;
    char msg[500];
    struct sockaddr_in temp;
    int templen;

    myhost[0] = '\0';
    get_qualified_hostname(myhost);
    /* gethostname(myhost, sizeof(myhost)); */

    conn = connect_to_server(host);
    if (conn < 0)
	return -1;

#ifdef DEBUG
    printf("Connected\n");
#endif

    pw = getpwuid(geteuid());
    if (pw == NULL)
    {
	extern char *getlogin();

	local_username = getlogin();
	if (local_username == NULL)
	{
	    start_prog_error = "Cannot get pw entry";
	    return -3;
	}
    }
    else
    {
	local_username = pw->pw_name;
    }

    send_string(conn, local_username);
    send_string(conn, username);
    recv_string(conn, buf, sizeof(buf));
#ifdef DEBUG
    printf("Got reply1 '%s'\n", buf);
#endif

    if (strncmp(buf, "Password", 8) == 0)
    {
	if (pw_hook == NULL)
	    pw_string = "";
	else
	    pw_string = (*pw_hook) (host, username);
	send_string(conn, pw_string);
	recv_string(conn, buf, sizeof(buf));
#ifdef DEBUG
	printf("Got reply '%s'\n", buf);
#endif
	if (strncmp(buf, "Proceed", 7) != 0)
	{
	    start_prog_error = buf;
	    return -4;
	}
    }
    else if (strncmp(buf, "Proceed", 7) != 0)
    {
	start_prog_error = buf;
	return -4;
    }

    sprintf(port_string, "%s %d %s", myhost, port, am_slave);
    send_string(conn, prog);
    send_string(conn, port_string);

    if (fork() == 0)
    {
	net_setup_anon_listener(10, &new_port, &new_fd);
	sprintf(port_string, "%d", new_port);
	fflush(stdout);
	send_string(conn, port_string);
	fflush(stdout);
	/* stdout_fd = net_accept(new_fd); */
	templen = sizeof(temp);
	stdout_fd = accept(new_fd, (struct sockaddr *) &temp, &templen);
	close(new_fd);
	while ((n = read(stdout_fd, msg, 499)) > 0)
	{
	    if (n > 0)
	    {
		write(1,msg,n);
		fflush(stdout);
	    }
	}
	exit(0);
    }

    recv_string(conn, buf, sizeof(buf));
#ifdef DEBUG
    printf("Got reply2 '%s'\n", buf);
#endif
    if (strncmp(buf, "Success", 7) != 0)
    {
	start_prog_error = buf;
	return -4;
    }
    start_prog_error = buf;
    close(conn);

    return 0;
}

static int connect_to_server(host)
char *host;
{
    int conn;
    int rc;
    struct hostent *hostent;
    struct sockaddr_in addr;

    conn = socket(AF_INET, SOCK_STREAM, 0);
    if (conn < 0)
    {
	start_prog_error = sys_errlist[errno];
	return -1;
    }

    hostent = gethostbyname_p4(host);

    addr.sin_family = hostent->h_addrtype;
    addr.sin_port = htons(sserver_port);
    bcopy(hostent->h_addr, &addr.sin_addr, hostent->h_length);

    rc = connect(conn, (struct sockaddr *) & addr, sizeof(addr));
    if (rc < 0)
    {
	start_prog_error = sys_errlist[errno];
	return -1;
    }

    return conn;
}

static void send_string(sock, str)
int sock;
char *str;
{
    int len = strlen(str);
    char nl = 10;

    if (write(sock, str, len) < 0)
    {
	perror("write");
    }
    if (write(sock, &nl, 1) < 0)
    {
	perror("write");
    }

}

static void recv_string(sock, buf, len)
int sock, len;
char *buf;
{
    char *bptr;
    int n;

    bptr = buf;
    while (1)
    {
	n = read(sock, bptr, 1);
	if (n < 0)
	{
	    perror("read");
	    exit(1);
	}
	if (*bptr == '\n')
	    break;
	bptr++;
	if (bptr - buf >= len)
	    break;
    }
    *bptr = 0;
}

#ifdef P4BSD

#include <sys/ioctl.h>
static struct sgttyb orig_tty;

static int echo_off()
{
    struct sgttyb tty_new;

    if (ioctl(0, TIOCGETP, &orig_tty) < 0)
    {
	fprintf(stderr, "iotcl TIOCGETP failed: %s\n", sys_errlist[errno]);
	return -1;
    }

    tty_new = orig_tty;
    tty_new.sg_flags &= ~(ECHO);

    if (ioctl(0, TIOCSETP, &tty_new) < 0)
    {
	fprintf(stderr, "iotcl TIOCSETP failed: %s\n", sys_errlist[errno]);
	return -1;
    }
}

static int echo_on()
{
    if (ioctl(0, TIOCSETP, &orig_tty) < 0)
    {
	fprintf(stderr, "iotcl TIOCSETP failed: %s\n", sys_errlist[errno]);
	return -1;
    }
}

#else

#include <termio.h>

struct termio tty_orig;

static int echo_off()
{
    struct termio tty_new;

    if (ioctl(0, TCGETA, &tty_orig) < 0)
    {
	fprintf(stderr, "tcgetattr failed: %s\n", sys_errlist[errno]);
	return -1;
    }

    tty_new = tty_orig;

    tty_new.c_lflag &= ~(ECHO);

    if (ioctl(0, TCSETA, &tty_new) < 0)
    {
	fprintf(stderr, "tcsetattr failed: %s\n", sys_errlist[errno]);
	return -1;
    }
}

static int echo_on()
{
    if (ioctl(0, TCSETA, &tty_orig) < 0)
    {
	fprintf(stderr, "tcsetattr failed: %s\n", sys_errlist[errno]);
	return -1;
    }
}

#endif

char *getpw(host, name)
char *host, *name;
{
    static char buf[1024];
    char *s;

    echo_off();
    printf("Password for %s@%s: ", name, host);
    fflush(stdout);
    fgets(buf, sizeof(buf), stdin);
    echo_on();
    printf("\n");

    for (s = buf; *s; s++)
	if (*s == '\n')
	{
	    *s = 0;
	    break;
	}

    return buf;
}

#endif
/* #ifdef SYMMETRY */
SHAR_EOF
if test -f 'p4_shmem_sr.c'
then
	echo shar: over-writing existing file "'p4_shmem_sr.c'"
fi
cat << \SHAR_EOF > 'p4_shmem_sr.c'
#include "p4.h"
#include "p4_sys.h"

P4BOOL shmem_msgs_available()
{
    int rc, qidx;
    struct p4_msg_queue *mq;

    qidx = p4_local->my_id - p4_global->low_cluster_id;
    mq = &(p4_global->shmem_msg_queues[qidx]);
    rc = (mq->first_msg != NULL);
    return (rc);
}


struct p4_msg *shmem_recv()
{
    struct p4_msg_queue *mq;
    struct p4_queued_msg *q;
    struct p4_msg *m;
    int my_qidx, from_qidx;

    my_qidx = p4_local->my_id - p4_global->low_cluster_id;
    mq = &(p4_global->shmem_msg_queues[my_qidx]);
    p4_dprintfl(60, "receiving shmem messages %d\n", my_qidx);

    p4_menter(&mq->m);
    if (mq->first_msg == NULL)
    {
	p4_mdelay(&mq->m, 0);
    }
    q = mq->first_msg;
    if (mq->first_msg == mq->last_msg)
    {
	mq->first_msg = NULL;
	mq->last_msg = NULL;
    }
    else
	mq->first_msg = mq->first_msg->next;
    p4_mcontinue(&mq->m, 0);

    from_qidx = q->qmsg->from - p4_global->low_cluster_id;
    if (q->qmsg->ack_req & P4_ACK_REQ_MASK)
    {
	p4_dprintfl(30, "sending ack to %d\n", q->qmsg->from);
	p4_unlock(&(p4_global->shmem_msg_queues[from_qidx].ack_lock));
	p4_dprintfl(30, "sent ack to %d\n", q->qmsg->from);
    }

    m = q->qmsg;
    free_quel(q);
    p4_dprintfl(60, "received from %d via shmem\n", q->qmsg->from);
    return (m);
}				/* shmem_recv */


int shmem_send(tmsg)
struct p4_msg *tmsg;
{

    struct p4_msg_queue *mq;
    int from, to_qidx, from_qidx;

    p4_dprintfl(20, "sending msg of type %d from %d to %d via shmem\n",tmsg->type,tmsg->from,tmsg->to);
    to_qidx = tmsg->to - p4_global->low_cluster_id;
    from_qidx = tmsg->from - p4_global->low_cluster_id;
    mq = &(p4_global->shmem_msg_queues[to_qidx]);

    p4_menter(&mq->m);
    queue_p4_message(tmsg, mq);
    p4_mcontinue(&mq->m, 0);

    if (tmsg->ack_req & P4_ACK_REQ_MASK)
    {
	p4_dprintfl(30, "waiting for ack from %d\n", tmsg->to);
	p4_lock(&(p4_global->shmem_msg_queues[from_qidx].ack_lock));
	p4_dprintfl(30, "received ack from %d\n", tmsg->to);
    }
    p4_dprintfl(10, "sent msg of type %d from %d to %d via shmem\n",tmsg->type,tmsg->from,tmsg->to);

    return (0);
}
SHAR_EOF
if test -f 'p4_sr.h'
then
	echo shar: over-writing existing file "'p4_sr.h'"
fi
cat << \SHAR_EOF > 'p4_sr.h'
struct p4_msg_queue {
    p4_monitor_t m;
    p4_lock_t ack_lock;
    struct p4_queued_msg *first_msg, *last_msg;
};

#define CONN_ME 		1
#define CONN_REMOTE_SWITCH 	2
#define CONN_REMOTE_NON_EST 	3
#define CONN_REMOTE_EST 	4
#define CONN_SHMEM	 	5
#define CONN_CUBE 		6
#define CONN_TCMP 		7
#define CONN_REMOTE_DYING       8

#define P4_MAX_MSGLEN (1<<26)  /* Used in free_p4_msg as sanity check
				  increase as desired */

/* Returns true if node is out of allowed range */
#define CHECKNODE(node) ((node<0) || (node>=p4_num_total_ids()))

#if defined(IPSC860) || defined(CM5)
#    define CONN_LOCAL CONN_CUBE
#else
#    if defined(TCMP)
#        define CONN_LOCAL CONN_TCMP
#    else
#        define CONN_LOCAL CONN_SHMEM
#    endif
#endif

#define XDR_PAD 4
#define XDR_INT_LEN 4
#define XDR_LNG_LEN 4
#define XDR_FLT_LEN 4
#define XDR_DBL_LEN 8
#define XDR_CHR_LEN 4
#define XDR_BUFF_LEN 4096

/* data types typically used by xdr, but also for other purposes */
#define P4NOX 0
#define P4INT 1
#define P4LNG 2
#define P4FLT 3
#define P4DBL 4


/* send_message(type,from,to,msg,len,data_type,ack_req,p4_buff_ind) */

#define  p4_send(TYPE,TO,MSG,LEN)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,P4NOX,FALSE,FALSE)
#define  p4_sendr(TYPE,TO,MSG,LEN)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,P4NOX,TRUE,FALSE)
#define  p4_sendx(TYPE,TO,MSG,LEN,DATATYPE)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,DATATYPE,FALSE,FALSE)
#define  p4_sendrx(TYPE,TO,MSG,LEN,DATATYPE)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,DATATYPE,TRUE,FALSE)
#define  p4_sendb(TYPE,TO,MSG,LEN)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,P4NOX,FALSE,TRUE)
#define  p4_sendbr(TYPE,TO,MSG,LEN)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,P4NOX,TRUE,TRUE)
#define  p4_sendbx(TYPE,TO,MSG,LEN,DATATYPE)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,DATATYPE,FALSE,TRUE)
#define  p4_sendbrx(TYPE,TO,MSG,LEN,DATATYPE)  \
    send_message(TYPE,p4_get_my_id(),TO,MSG,LEN,DATATYPE,TRUE,TRUE)

#define  p4_broadcast(TYPE,MSG,LEN)  p4_broadcastx(TYPE,MSG,LEN,P4NOX)
SHAR_EOF
if test -f 'p4_tsr.c'
then
	echo shar: over-writing existing file "'p4_tsr.c'"
fi
cat << \SHAR_EOF > 'p4_tsr.c'
#include "p4.h"
#include "p4_sys.h"

/*
 * search_p4_queue tries to locate a message of the desired type in the
 * local queue of messages already received.  If it finds one, it dequeues it 
 * if deq is true, and returns its address; otherwise it returns NULL.
 */
struct p4_msg *search_p4_queue(req_type, req_from, deq)
int req_type, req_from;
P4BOOL deq;
{
    struct p4_queued_msg *qpp, *qp;
    struct p4_msg *tqp;
    P4BOOL found;

    tqp = NULL;
    qpp = NULL;
    found = FALSE;
    qp = p4_local->queued_messages->first_msg;

    while (qp && !found)
    {
	if (((qp->qmsg->type == req_type) || (req_type == -1)) &&
	    ((qp->qmsg->from == req_from) || (req_from == -1)))
	{
	    found = TRUE;
	    if (deq)
	    {
		if (p4_local->queued_messages->first_msg ==
		    p4_local->queued_messages->last_msg)
		{
		    p4_local->queued_messages->first_msg = NULL;
		    p4_local->queued_messages->last_msg = NULL;
		}
		else
		{
		    if (qp == p4_local->queued_messages->first_msg)
		    {
			p4_local->queued_messages->first_msg = qp->next;
		    }
		    else if (qp == p4_local->queued_messages->last_msg)
		    {
			p4_local->queued_messages->last_msg = qpp;
			qpp->next = NULL;
		    }
		    else
		    {
			qpp->next = qp->next;
		    }
		}
	    }
	}
	else
	{
	    qpp = qp;
	    qp = qp->next;
	}
    }
    if (found)
    {
	p4_dprintfl(30,"extracted queued msg of type %d from %d\n",
		    qp->qmsg->type,qp->qmsg->from);
	tqp = qp->qmsg;
	if (deq)
	{
	    free_quel(qp);
	}
    }
    return (tqp);
}

/*
 * This is the top-level receive routine, called by the user.
 *   req_type is either a desired type or -1.  In the -1 case it will be
 *        modified  by p4_recv to indicate the type actually received.
 *   req_from is either a desired source or -1.  In the -1 case it will be
 *        modified by p4_recv to the source of the message actually received.
 *   msg will be set by p4_recv to point to a buffer containing the message.
 *   len_rcvd will be set by p4_recv to contain the length of the message.
 *
 *   returns 0 if successful; non-zero if error
 */
int p4_recv(req_type, req_from, msg, len_rcvd)
int *req_type, *req_from, *len_rcvd;
char **msg;
{
    struct p4_msg *tmsg;
    P4BOOL good;

    p4_dprintfl(20, "receiving for type = %d, sender = %d\n",
		*req_type, *req_from);
    ALOG_LOG(p4_local->my_id,END_USER,0,"");
    ALOG_LOG(p4_local->my_id,BEGIN_RECV,*req_from,"");
    if (!(tmsg = search_p4_queue(*req_type, *req_from, 1)))
    {
	for (good = FALSE; !good;)
	{
            ALOG_LOG(p4_local->my_id,END_RECV,0,"");
            ALOG_LOG(p4_local->my_id,BEGIN_WAIT,0,"");
            tmsg = recv_message(req_type, req_from);
            ALOG_LOG(p4_local->my_id,END_WAIT,0,"");
            ALOG_LOG(p4_local->my_id,BEGIN_RECV,0,"");
	    if (tmsg == NULL)
	    {
		p4_dprintf("p4_recv: could not receive a message\n");
		return (-1);
	    }
	    if (((tmsg->type == *req_type) || (*req_type == -1)) &&
		((tmsg->from == *req_from) || (*req_from == -1)))
	    {
		good = TRUE;
	    }
	    else
	    {
		queue_p4_message(tmsg, p4_local->queued_messages) ;
	    }
	}
    }

    if (tmsg->ack_req & P4_BROADCAST_MASK)
    {
	if (subtree_broadcast_p4(tmsg->type, tmsg->from,(char *) &tmsg->msg,
				 tmsg->len, tmsg->data_type))
	{
	    p4_dprintf("p4_recv: subtree_brdcst failed\n");
	    return -1;
	}
	tmsg->ack_req &= ~P4_BROADCAST_MASK;	/* Unset broadcast bit */
	if (tmsg->from == p4_get_my_id())
	    free_p4_msg(tmsg);	/* Don't want broadcast from self */
    }

    *req_type = tmsg->type;
    *req_from = tmsg->from;
    p4_dprintfl(10, "received type=%d, from=%d\n",*req_type,*req_from);
    if (*msg == NULL)
    {
	*msg = (char *) &(tmsg->msg);
	*len_rcvd = tmsg->len;
    }
    else
    {
	/* copy into the user's buffer area, truncating if necessary */
	if (tmsg->len < *len_rcvd)
	    *len_rcvd = tmsg->len;
	bcopy((char *) &(tmsg->msg),*msg,*len_rcvd);
	free_p4_msg(tmsg);
    }
    ALOG_LOG(p4_local->my_id,END_RECV,*req_from,"");
    ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");

    return (0);
}

struct p4_msg *recv_message(req_type,req_from)
int *req_type, *req_from;
{
    int rc, len;
    struct p4_msg *tmsg;

#if  defined(CAN_DO_SOCKET_MSGS) && \
    !defined(CAN_DO_SHMEM_MSGS)  && \
    !defined(CAN_DO_CUBE_MSGS)   && \
    !defined(CAN_DO_SWITCH_MSGS)

    return (socket_recv());

#else

    while (TRUE)
    {
#       if defined(CAN_DO_SHMEM_MSGS)
	if (shmem_msgs_available())
	{
	    return (shmem_recv());
	}
#       endif

#       if defined(CAN_DO_SOCKET_MSGS)
	if (socket_msgs_available())
	{
	    return (socket_recv());
	}
#       endif

#       if defined(CAN_DO_CUBE_MSGS)
	if (MD_cube_msgs_available())
	    return (MD_cube_recv());
#       endif

#       if defined(CAN_DO_SWITCH_MSGS)
	if (p4_global->proctable[p4_local->my_id].switch_port != -1)
	{
	    if (rc = sw_probe(req_from, p4_local->my_id, req_type, &len))
	    {
		tmsg = alloc_p4_msg(len - sizeof(struct p4_msg) + sizeof(char *));
		sw_recv(rc, tmsg);
		p4_dprintfl(10, "p4_recv: received message from switch\n");
		return (tmsg);
	    }
	}
#       endif
#       if defined(CAN_DO_TCMP_MSGS)
	if (MD_tcmp_msgs_available(req_type,req_from))
	{
	    return (MD_tcmp_recv());
	}
#       endif
    }

#endif
}

P4BOOL p4_messages_available(req_type, req_from)
int *req_type, *req_from;
{
    int found, len;
    struct p4_msg *tmsg;

    ALOG_LOG(p4_local->my_id,END_USER,0,"");
    ALOG_LOG(p4_local->my_id,BEGIN_WAIT,1,"");

    found = FALSE;
    if (tmsg = search_p4_queue(*req_type, *req_from, 0))
    {
	found = TRUE;
	*req_type = tmsg->type;
	*req_from = tmsg->from;
    }

#   if defined(CAN_DO_SHMEM_MSGS)
    while (!found && shmem_msgs_available())
    {
	tmsg = shmem_recv();
	if (((tmsg->type == *req_type) || (*req_type == -1)) &&
	    ((tmsg->from == *req_from) || (*req_from == -1)))
	{
	    found = TRUE;
	    *req_type = tmsg->type;
	    *req_from = tmsg->from;
	}
	queue_p4_message(tmsg, p4_local->queued_messages);
    }
#   endif

#   if defined(CAN_DO_SOCKET_MSGS)
    while (!found && socket_msgs_available())
    {
	tmsg = socket_recv();
	if (((tmsg->type == *req_type) || (*req_type == -1)) &&
	    ((tmsg->from == *req_from) || (*req_from == -1)))
	{
	    found = TRUE;
	    *req_type = tmsg->type;
	    *req_from = tmsg->from;
	}
	queue_p4_message(tmsg, p4_local->queued_messages);
    }
#   endif

#   if defined(CAN_DO_CUBE_MSGS)
    while (!found && MD_cube_msgs_available())
    {
	tmsg = MD_cube_recv();
	if (((tmsg->type == *req_type) || (*req_type == -1)) &&
	    ((tmsg->from == *req_from) || (*req_from == -1)))
	{
	    found = TRUE;
	    *req_type = tmsg->type;
	    *req_from = tmsg->from;
	}
	queue_p4_message(tmsg, p4_local->queued_messages);
    }
#   endif


#if defined(CAN_DO_SWITCH_MSGS)
    if (!found && (p4_global->proctable[p4_local->my_id].switch_port != -1))
    {
	if (sw_probe(req_from, p4_local->my_id, req_type, &len))
	    found = TRUE;
    }
#endif

#if defined(CAN_DO_TCMP_MSGS)
    if (!found && MD_tcmp_msgs_available(req_from,req_type))
	found = TRUE;
#endif

    ALOG_LOG(p4_local->my_id,END_WAIT,1,"");
    ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");

    return (found);
}				/* p4_messages_available */

P4VOID queue_p4_message(msg, hdr)
struct p4_msg *msg;
struct p4_msg_queue *hdr;
{
    struct p4_queued_msg *q;

    q = alloc_quel();
    q->qmsg = msg;
    q->next = NULL;

    if (hdr->first_msg == NULL)
    {
	hdr->first_msg = q;
    }
    else
    {
	hdr->last_msg->next = q;
    }
    hdr->last_msg = q;
    p4_dprintfl(30,"queued type %d message for process %d quel=%d\n",
		msg->type,msg->to,q);
}


int send_message(type, from, to, msg, len, data_type, ack_req, p4_buff_ind)
char *msg;
int type, to, len, data_type;
P4BOOL ack_req, p4_buff_ind;
{
    struct p4_msg *tmsg;
    int conntype = p4_local->conntab[to].type;

    p4_dprintfl(90, "send_message: to = %d, conntype=%d conntype=%s\n",
		to, conntype, print_conn_type(conntype));

    switch (conntype)
    {
      case CONN_ME:
	tmsg = get_tmsg(type,from,to,msg,len,data_type,ack_req,p4_buff_ind);
	p4_dprintfl(20, "sending msg of type %d to myself\n",type);
	queue_p4_message(tmsg, p4_local->queued_messages);
	p4_dprintfl(10, "sent msg of type %d to myself\n",type);
	break;

#ifdef CAN_DO_SHMEM_MSGS
      case CONN_SHMEM:
	tmsg = get_tmsg(type, from, to, msg, len, data_type, 
                        ack_req, p4_buff_ind);
	shmem_send(tmsg);
	break;
#endif

#ifdef CAN_DO_CUBE_MSGS
      case CONN_CUBE:
	tmsg = get_tmsg(type,from,to,msg,len,data_type,ack_req,p4_buff_ind);
	MD_cube_send(tmsg);
	if (!p4_buff_ind)
	    free_p4_msg(tmsg);
	break;
#endif

#ifdef CAN_DO_SOCKET_MSGS
      case CONN_REMOTE_NON_EST:
	if (establish_connection(to))
	{
	    p4_dprintfl(90, "send_message: conn just estabd to %d\n", to);
	}
	else
	{
	    p4_dprintf("send_message: unable to estab conn to %d\n", to);
	    return (-1);
	}
	/* no break; - just fall into connected code */
      case CONN_REMOTE_EST:
	if (data_type == P4NOX || p4_local->conntab[to].same_data_rep)
	{
	    socket_send(type, from, to, msg, len, data_type, ack_req);
	}
	else
	{
#           ifdef CAN_DO_XDR
	    xdr_send(type, from, to, msg, len, data_type, ack_req);
#           else
	    p4_error("cannot do xdr sends\n",0);
#           endif
	}
	break;
#endif

#if defined(CAN_DO_SWITCH_MSGS)
      case CONN_REMOTE_SWITCH:
	tmsg = get_tmsg(type,from,to,msg,len,data_type,ack_req,p4_buff_ind);
	p4_dprintfl(20, "sending msg of type %d from %d to %d via switch_port %d\n",
	            tmsg->type,tmsg->from,to,p4_local->conntab[tmsg->to].switch_port,tmsg);
	sw_send(from, to,
		p4_local->conntab[tmsg->to].switch_port, tmsg,
		tmsg->len + sizeof(struct p4_msg) - sizeof(char *),
		type);
	p4_dprintfl(10, "sent msg of type %d from %d to %d via switch_port %d\n",
	            tmsg->type,tmsg->from,to,p4_local->conntab[tmsg->to].switch_port,tmsg);
	if (!p4_buff_ind)
	    free_p4_msg(tmsg);
	break;
#endif

#if defined(CAN_DO_TCMP_MSGS)
      case CONN_TCMP:
	tmsg = get_tmsg(type,from,to,msg,len,data_type,ack_req,p4_buff_ind);
	p4_dprintfl(20, "sending msg of type %d to %d via tcmp\n",type,to);
	MD_tcmp_send(type, from, to, tmsg, 
		     tmsg->len + sizeof(struct p4_msg) - sizeof(char *),
		     data_type, ack_req);
	p4_dprintfl(10, "sent msg of type %d to %d via tcmp\n",type,to);
	break;
#endif

      case CONN_REMOTE_DYING:
	p4_dprintfl(90, "send_message: proc %d is dying\n", tmsg->to);
	return (-1);

      default:
	p4_dprintf("send_message: to=%d; invalid conn type=%d\n",to,conntype);
	return (-1);
    }

    return (0);
}				/* send_message */

struct p4_msg *get_tmsg(type,from,to,msg,len,data_type,ack_req,p4_buff_ind)
char *msg;
int type, from, to, len, data_type, ack_req, p4_buff_ind;
{
    int i;
    struct p4_msg *tmsg;

    if (p4_buff_ind)
    {
	tmsg = (struct p4_msg *) (msg - sizeof(struct p4_msg) + sizeof(char *));
    }
    else
    {
        tmsg = alloc_p4_msg(len);
	if (tmsg == NULL)
	{
	    p4_dprintf("OOPS! get_tmsg: could not alloc buff **\n");
	    return (NULL);
	}
	bcopy(msg, (char *) &(tmsg->msg), len);
    }
    tmsg->type = type;
    tmsg->from = from;
    tmsg->to = to;
    tmsg->len = len;
    tmsg->ack_req = ack_req;
    tmsg->data_type = data_type;
    return (tmsg);
}


char *p4_msg_alloc(msglen)
int msglen;
{
    char *t;

    t = (char *) alloc_p4_msg(msglen);
    ((struct p4_msg *) t)->msg_id = -1;	/* msg not in use by IPSC isend */
    t = t + sizeof(struct p4_msg) - sizeof(char *);
    return(t);
}

P4VOID p4_msg_free(m)
char *m;
{
    char *t;

    t = m - sizeof(struct p4_msg) + sizeof(char *);
    ((struct p4_msg *) t)->msg_id = -1;	/* msg not in use by IPSC isend */
    free_p4_msg((struct p4_msg *) t);
}


P4VOID initialize_msg_queue(mq)
struct p4_msg_queue *mq;
{
    mq->first_msg = NULL;
    mq->last_msg = NULL;
    (P4VOID) p4_moninit(&(mq->m), 1);
    p4_lock_init(&(mq->ack_lock));
    p4_lock(&(mq->ack_lock));
}


struct p4_queued_msg *alloc_quel()
{
    struct p4_queued_msg *q;

    p4_lock(&p4_global->avail_quel_lock);
    if (p4_global->avail_quel == NULL)
    {
	q = (struct p4_queued_msg *) p4_shmalloc(sizeof(struct p4_queued_msg));
	if (!q)
	    p4_error("alloc_quel:  could not allocate queue element",
		     sizeof(struct p4_queued_msg));
    }
    else
    {
	q = p4_global->avail_quel;
	p4_global->avail_quel = q->next;
    }
    p4_unlock(&p4_global->avail_quel_lock);
    return (q);
}

P4VOID free_quel(q)
struct p4_queued_msg *q;
{
    p4_lock(&p4_global->avail_quel_lock);
    q->next = p4_global->avail_quel;
    p4_global->avail_quel = q;
    p4_unlock(&p4_global->avail_quel_lock);
}
SHAR_EOF
if test -f 'p4_sock_conn.c'
then
	echo shar: over-writing existing file "'p4_sock_conn.c'"
fi
cat << \SHAR_EOF > 'p4_sock_conn.c'
#include "p4.h"
#include "p4_sys.h"

int establish_connection(dest_id)
int dest_id;
{
    int myid = p4_get_my_id();

    p4_global->dest_id[myid] = dest_id;
    request_connection(dest_id);
    p4_global->dest_id[myid] = (-1);

    if (myid > dest_id)
    {
	/* following should not spin long */
	while (p4_local->conntab[dest_id].type == CONN_REMOTE_NON_EST)
	    p4_dprintfl(70, "waiting for interrupt handler to do its job\n");
    }
    return (TRUE);
}


P4VOID request_connection(dest_id)
int dest_id;
{
    struct proc_info *my_pi, *dest_pi;
    char *my_host, *dest_host;
    int my_id;
    struct slave_listener_msg msg;
    int connection_fd;
    int dest_listener_con_fd;
    int connected_to_dest_listener;
    int my_listener, dest_listener;
    int new_listener_port, new_listener_fd;
    int oldmask;
    int num_tries;

#   ifdef P4SYSV
    sighold(LISTENER_ATTN_SIGNAL);
#   else
    oldmask = sigblock(sigmask(LISTENER_ATTN_SIGNAL));
#   endif

    /* Get some initial information */
    my_id = p4_get_my_id();
    my_pi = get_proc_info(my_id);
    my_host = my_pi->host_name;
    my_listener = my_pi->port;

    dest_pi = get_proc_info(dest_id);
    dest_host = dest_pi->host_name;
    dest_listener = dest_pi->port;

    p4_dprintfl(70, "request_connection: my_id=%d my_host=%s my_listener=%d dest_id=%d dest_host=%s dest_listener=%d\n",
	    my_id, my_host, my_listener, dest_id, dest_host, dest_listener);

    /* Have we already connected?? */
    if (p4_local->conntab[dest_id].type == CONN_REMOTE_EST)
    {
	p4_dprintf("request_connection %d: already connected\n", dest_id);
#       ifdef P4SYSV
        sigrelse(LISTENER_ATTN_SIGNAL);
#       else
	sigsetmask(oldmask);
#       endif
	return;
    }

    p4_dprintfl(70, "enter loop to connect to dest listener %s\n",dest_host);
    /* Connect to dest listener */
    num_tries = 1;
    while((dest_listener_con_fd = net_conn_to_listener(dest_host,dest_listener,1)) == -1)
	num_tries++;
    p4_dprintfl(70, "conn_to_proc_contd: connected after %d tries, dest_listener_con_fd=%d\n",num_tries, dest_listener_con_fd);


    /* Setup a listener to accept the connection to dest_id */
    net_setup_anon_listener(1, &new_listener_port, &new_listener_fd);

    /* Construct a connection request message */
    msg.type = p4_i_to_n(CONNECTION_REQUEST);
    msg.from = p4_i_to_n(my_id);
    msg.lport = p4_i_to_n(new_listener_port);
    msg.to = p4_i_to_n(dest_id);
    msg.to_pid = p4_i_to_n(dest_pi->unix_id);

    /* Send it to dest_id's listener */
    p4_dprintfl(70, "request_connection: sending CONNECTION_REQUEST to %d on fd=%d size=%d\n",
		dest_id,dest_listener_con_fd,sizeof(msg));
    net_send(dest_listener_con_fd, &msg, sizeof(msg), FALSE);
    p4_dprintfl(70, "request_connection: sent CONNECTION_REQUEST to dest_listener\n");

    if (my_id < dest_id)
    {
	/* Wait for the remote process to connect to me */
	p4_dprintfl(70, "request_connection: waiting for accept from %d \n",dest_id);
	connection_fd = net_accept(new_listener_fd);
	p4_dprintfl(70, "request_connection: accepted from %d on %d\n", dest_id, connection_fd);

	/* Add the connection to the table */
	p4_local->conntab[dest_id].type = CONN_REMOTE_EST;
	p4_local->conntab[dest_id].port = connection_fd;
	p4_local->conntab[dest_id].same_data_rep =
	    same_data_representation(p4_local->my_id,dest_id);
    }

    close(dest_listener_con_fd);
    /* Now release the listener connections */
    close(new_listener_fd);

#   ifdef P4SYSV
    sigrelse(LISTENER_ATTN_SIGNAL);
#   else
    sigsetmask(oldmask);
#   endif

    p4_dprintfl(70, "request_connection: finished connecting\n");
    return;
}

P4VOID handle_connection_interrupt()
{
    struct slave_listener_msg msg;
    int type;
    int listener_fd;
    int to, to_pid, from, lport;
    int connection_fd;
    struct proc_info *from_pi;
    int myid = p4_get_my_id();
    int num_tries;

    listener_fd = p4_local->listener_fd;
    p4_dprintfl(70, "Inside handle_connection_interrupt, listener_fd=%d\n",
		listener_fd);

    if (net_recv(listener_fd, &msg, sizeof(msg)) == PRECV_EOF)
    {
	p4_dprintf("OOPS: got eof in handle_connection_interrupt\n");
	return;
    }

    type = p4_n_to_i(msg.type);
    if (type != CONNECTION_REQUEST)
    {
	p4_dprintf("handle_connection_interrupt: invalid type %d\n", type);
	return;
    }

    to = p4_n_to_i(msg.to);
    from = p4_n_to_i(msg.from);
    to_pid = p4_n_to_i(msg.to_pid);
    lport = p4_n_to_i(msg.lport);

    p4_dprintfl(70, "handle_connection_interrupt: msg contents: to=%d from=%d to_pid=%d lport=%d\n",
		to, from, to_pid, lport);

    /* If we're already connected, forget about the interrupt. */
    if (p4_local->conntab[from].type != CONN_REMOTE_EST)
    {
	if (myid < from)
	{
	    /* see if I have already started this connection */
	    p4_dprintfl(90,"myid < from, myid = %d, from = %d\n",myid,from);
	    if (p4_global->dest_id[myid] != from)
		request_connection(from);
	}
	else
	{
	    /* Get the information for the process we're connecting to */
	    from_pi = &(p4_global->proctable[from]);

	    /* Connect to the waiting process */
	    p4_dprintfl(70, "connecting to port...\n");
	    num_tries = 1;
	    /* connect to the requesting process, who is listening */
	    p4_dprintfl(70,"handling connection interrupt: connecting to %s\n",from_pi->host_name);
	    while ((connection_fd = net_conn_to_listener(from_pi->host_name,lport,1)) == -1)
		num_tries++;

	    p4_dprintfl(70, "handling connection interrupt: connected after %d tries, connection_fd=%d host = %s\n",
			num_tries, connection_fd, from_pi->host_name);

	    /* We're connected, so we can add this connection to the table */
	    p4_local->conntab[from].type = CONN_REMOTE_EST;
	    p4_local->conntab[from].port = connection_fd;
	    p4_local->conntab[from].same_data_rep =
		same_data_representation(p4_local->my_id,from);
	    p4_dprintfl(70, "marked as live fd=%d from=%d\n", connection_fd, from);
	}
    }
    else
    {
	p4_dprintfl(70,"ignoring interrupt from %d\n",from);
    }

    msg.type = p4_i_to_n(IGNORE_THIS);
    p4_dprintfl(70, "handle_connection_interrupt: sending IGNORE_THIS to my_listener\n");
    /* send msg to listener indicating I made the connection */
    net_send(listener_fd, &msg, sizeof(msg), FALSE);
    p4_dprintfl(70, "handle_connection_interrupt: exiting handling intr from %d\n",from);
    SIGNAL_P4(LISTENER_ATTN_SIGNAL, handle_connection_interrupt);
}
SHAR_EOF
if test -f 'p4_sock_cr.c'
then
	echo shar: over-writing existing file "'p4_sock_cr.c'"
fi
cat << \SHAR_EOF > 'p4_sock_cr.c'
#include "p4.h"
#include "p4_sys.h"

int create_remote_processes(pg)
struct p4_procgroup *pg;
{
    struct p4_procgroup_entry *pe;
    int i, rm_num = 1;

    for (i = 0, pe = pg->entries; i < pg->num_entries; i++, pe++)
    {
	if (strcmp(pe->host_name, "local") != 0)
	{
	    create_one_remote(pe, rm_outfile_head, rm_num++);
	}
    }

    return (0);
}

P4VOID create_one_remote(pe, outfile, rm_num)
struct p4_procgroup_entry *pe;
char *outfile;
int rm_num;
{
    int rm_fd;
    struct bm_rm_msg msg;
    P4BOOL done;
    int type;
    int status, port, remote_switch_port;
    int slave_idx, slave_pid, pidx, rm_ind;

    rm_fd = net_create_slave(pe->host_name,
			     pe->slave_full_pathname,
			     pe->username);

    msg.type = p4_i_to_n(INITIAL_INFO);
    msg.numinproctab = p4_i_to_n(p4_global->num_in_proctable);
    msg.numslaves = p4_i_to_n(pe->numslaves_in_group);
    strcpy(msg.outfile, outfile);
    msg.rm_num = p4_i_to_n(rm_num);
    msg.debug_level = p4_i_to_n(remote_debug_level);
    msg.memsize = p4_i_to_n(globmemsize);
    msg.logging_flag = p4_i_to_n(logging_flag);
    strcpy(msg.application_id, p4_global->application_id);
    strcpy(msg.version, p4_version());
    strcpy(msg.pgm, pe->slave_full_pathname);

    net_send(rm_fd, &msg, sizeof(msg), FALSE);

    port = -1;
    pidx = -1;
    for (done = FALSE; !done;)
    {
	status = net_recv(rm_fd, &msg, sizeof(msg));
	if (status == PRECV_EOF)
	{
	    p4_dprintf("OOPS! got EOF in create_one_remote\n");
	    return;
	}


	type = p4_n_to_i(msg.type);
	switch (type)
	{
	  case REMOTE_LISTENER_INFO:
	    port = p4_n_to_i(msg.port);
	    break;

	  case REMOTE_MASTER_INFO:
	  case REMOTE_SLAVE_INFO:
	    if (type == REMOTE_MASTER_INFO)
	       rm_ind = TRUE;
	    else
	       rm_ind = FALSE;
	    slave_idx = p4_n_to_i(msg.slave_idx);
	    slave_pid = p4_n_to_i(msg.slave_pid);
	    remote_switch_port = p4_n_to_i(msg.switch_port);
	    if (port == -1)
		p4_dprintf("OOPS! got slave_info w/o getting port first\n");
	    /* big master installing remote processes */
	    pidx = install_in_proctable(rm_num,port,slave_pid,
					msg.host_name,slave_idx,
					msg.machine_type,remote_switch_port);
            p4_dprintfl(90, "create_one_remote: adding connection: %d --> %d\n",
		        p4_get_my_id(), pidx);

            if (p4_local->conntab[pidx].type == CONN_REMOTE_SWITCH)
            {
	        p4_local->conntab[pidx].switch_port = remote_switch_port;
	        p4_local->conntab[pidx].port = rm_fd;
            }
            else if (p4_local->conntab[pidx].type == CONN_REMOTE_NON_EST)
            {
		if (type == REMOTE_MASTER_INFO)
		{
	            p4_local->conntab[pidx].type = CONN_REMOTE_EST;
	            p4_local->conntab[pidx].port = rm_fd;
		    p4_local->conntab[pidx].same_data_rep =
			same_data_representation(p4_local->my_id,pidx);
		}
            }
            else
            {
	        p4_error("create_one_remote: invalid conn type in conntab\n",
		         p4_local->conntab[pidx].type);
            }
	    break;

	  case REMOTE_SLAVE_INFO_END:
	    done = TRUE;
	    break;
	}
    }

}


/*
 *	Run the slave pgm on host; returns the file descriptor of the
 *	connection to the slave.
 */
int net_create_slave(host, pgm, username)
char *host;
char *pgm;
char *username;
{
    int serv_port, serv_fd;
    int slave_fd;
    int slave_pid, master_pid;
    char hostname[100];
    struct net_initial_handshake hs;
    char myhost[100];
    int connection_fd, success, rc;
    struct net_message_t msg;
    char remote_shell[64];
    char serv_port_c[64];
    char *am_slave_c = "-amp4slave";
#if defined(SYMMETRY) || defined(SUN) || defined(DEC5000) 
    char *getpw();
#endif

    net_setup_anon_listener(5, &serv_port, &serv_fd);

    myhost[0] = '\0';
    get_qualified_hostname(myhost);

    /* try to connect to (secure) server */

#if defined(SYMMETRY) || defined(SUN) || defined(DEC5000) 
    /*****  secure server stuff  *******/
    p4_dprintfl(20, "trying to create remote slave on %s via server\n",host);
    rc = start_slave(host, username, pgm, serv_port, am_slave_c, getpw);

    if (rc < -1)
    {
	extern char *start_prog_error;

	p4_dprintf("Error with secure server: %s\n", start_prog_error);
    }
    else if (rc == 0)
        p4_dprintfl(10, "created remote slave on %s via server\n",host);
    /*****************************************/
#else
    rc = -1;
#endif

    if (rc <= -1)
    {
	/* try to connect to (old) server */
	connection_fd = net_conn_to_listener(host, UNRESERVED_PORT, 1);

	if (connection_fd >= 0)
	{
	    p4_dprintfl(20, "creating remote slave on %s via old server\n",host);
	    msg.type = p4_i_to_n(NET_EXEC);
	    strcpy(msg.pgm, pgm);
	    strcpy(msg.host, myhost);
	    strcpy(msg.am_slave, am_slave_c);
	    msg.port = p4_i_to_n(serv_port);
	    net_send(connection_fd, &msg, sizeof(msg), FALSE);
	    net_recv(connection_fd, &msg, sizeof(msg));

	    success = p4_n_to_i(msg.success);
	    if (!success)
	    {
		p4_dprintf("create failed: %s\n", msg.message);
		return (-1);
	    }
	    close(connection_fd);
	    p4_dprintfl(10, "created remote slave on %s via old server\n",host);
	}
	else
	{
#if defined(DELTA)
	    p4_dprintf("delta cannot create remote processes\n");
#else
	    p4_dprintfl(20, "creating remote slave on %s via remote shell\n",host);
#ifdef P4BSD
	    strcpy(remote_shell, "rsh");
#endif

#ifdef P4SYSV
#    if defined(TITAN) || defined(SGI)
	    strcpy(remote_shell, "rsh");
#    else
#        if defined(SYMMETRY_PTX)
	    strcpy(remote_shell, "resh");
#        else
	    strcpy(remote_shell, "remsh");
#        endif
#    endif
#endif

	    sprintf(serv_port_c, "%d", serv_port);
	    rc = fork_p4();
	    if (rc == 0)
	    {
		rc = execlp(remote_shell, remote_shell,
			    host, "-l", username, "-n", pgm,
			    myhost, serv_port_c, am_slave_c, 0);
		/* host,"-n","cluster","5",pgm,myhost,serv_port_c,0); for butterfly */
		if (rc < 0)
		    p4_error("net_create_slave: execlp", rc);
	    }
	    p4_dprintfl(10, "created remote slave on %s via remote shell\n",host);
#endif
	}
    }
    slave_fd = net_accept(serv_fd);
    close(serv_fd);

    hs.pid = (int) htonl(getpid());
    net_send(slave_fd, &hs, sizeof(hs), FALSE);
    net_recv(slave_fd, &hs, sizeof(hs));
    slave_pid = (int) ntohl(hs.pid);

    return slave_fd;
}
SHAR_EOF
if test -f 'p4_sock_list.c'
then
	echo shar: over-writing existing file "'p4_sock_list.c'"
fi
cat << \SHAR_EOF > 'p4_sock_list.c'
#include "p4.h"
#include "p4_sys.h"

P4VOID listener()
{
    struct listener_data *l = listener_info;
    P4BOOL done = FALSE;
    fd_set read_fds;
    int i, nfds, fd;

    p4_dprintfl(70, "enter listener \n");
    dump_listener(70);

    while (!done)
    {
	FD_ZERO(&read_fds);
	FD_SET(l->listening_fd, &read_fds);
	FD_SET(l->slave_fd, &read_fds);

	nfds = select(p4_global->max_connections, &read_fds, 0, 0, 0);
	if (nfds < 0)
	    p4_error("listener select", nfds);
	if (nfds == 0)
	    p4_dprintfl(70, "select timeout\n");

	fd = 0;
	for (i = 0; i < nfds && !done; i++)
	{
	    while (fd < p4_global->max_connections)
	    {
		if (FD_ISSET(fd, &read_fds))
		{
		    if (fd == l->listening_fd || fd == l->slave_fd)
			break;
		}
		fd++;
	    }

	    p4_dprintfl(70, "got fd=%d listening_fd=%d slave_fd=%d\n",
			fd, l->listening_fd, l->slave_fd);

	    if (fd == l->listening_fd)
		done = process_connect_request(fd);
	    else
		done = process_slave_message(fd);
	    fd++;
	}
    }

    p4_dprintfl(70, "exit listener\n");
    exit(0);
}

P4BOOL process_connect_request(fd)
int fd;
{
    struct slave_listener_msg msg;
    int type;
    int connection_fd, slave_fd;
    int from, lport, to_pid, to;
    P4BOOL rc = FALSE;

    p4_dprintfl(70, "processing connect check/request on %d\n", fd);

    connection_fd = net_accept(fd);

    p4_dprintfl(70, "accepted on connection_fd=%d reading size=%d\n", connection_fd,sizeof(msg));

    if (net_recv(connection_fd, &msg, sizeof(msg)) == PRECV_EOF)
    {
	return (FALSE);
    }

    type = p4_n_to_i(msg.type);
    switch (type)
    {
      case IGNORE_THIS:
	p4_dprintfl(70, "got IGNORE_THIS\n");
	break;

      case CONNECTION_REQUEST:
	from = p4_n_to_i(msg.from);
	to_pid = p4_n_to_i(msg.to_pid);
	to = p4_n_to_i(msg.to);
	lport = p4_n_to_i(msg.lport);
	p4_dprintfl(70, "connection_request2: poking slave: from=%d lport=%d to_pid=%d to=%d\n",
		    from, lport, to_pid, to);

	slave_fd = listener_info->slave_fd;

	if (kill(to_pid, LISTENER_ATTN_SIGNAL) == -1)
	{
	    p4_dprintf("Listener: Unable to interrupt client pid=%d.\n", to_pid);
	    break;
	}

	net_send(slave_fd, &msg, sizeof(msg), FALSE);
	/* wait for msg from slave indicating it got connected */
	/*
	 * do not accept any more connections for slave until it has fully
	 * completed this one, i.e. do not want to interrupt it until it has
	 * handled this interrupt
	 */
	p4_dprintfl(70, "waiting for slave to handle interrupt\n");
	net_recv(slave_fd, &msg, sizeof(msg));
	p4_dprintfl(70, "back from slave handling interrupt\n");
	break;

      default:
	p4_dprintf("invalid type %d in process_connect_request\n", type);
	break;
    }
    close(connection_fd);
    return (rc);
}

P4BOOL process_slave_message(fd)
int fd;
{
    struct slave_listener_msg msg;
    int type;
    int from;
    P4BOOL rc = FALSE;
    int status;

    status = net_recv(fd, &msg, sizeof(msg));
    if (status == PRECV_EOF)
    {
	p4_error("slave_listener_msg: got eof on fd=", fd);
    }

    type = p4_n_to_i(msg.type);
    from = p4_n_to_i(msg.from);

    switch (type)
    {
      case DIE:
	p4_dprintfl(70, "received die msg from %d\n", from);
	rc = TRUE;
	break;

      default:
	p4_dprintf("received unknown message type=%d from=%d\n", type, from);
	p4_error("slave_listener_msg: unknown message type", type);
	break;
    }

    return (rc);
}
SHAR_EOF
if test -f 'p4_sock_sr.c'
then
	echo shar: over-writing existing file "'p4_sock_sr.c'"
fi
cat << \SHAR_EOF > 'p4_sock_sr.c'
#include "p4.h"
#include "p4_sys.h"

#ifdef CAN_DO_XDR
int xdr_send(type, from, to, msg, len, data_type, ack_req)
char *msg;
int type, from, to, len, data_type, ack_req;
{
    int done = 0;
    int nbytes_written = 0;
    int flag, sent, fd, myid, rc, n, nfds;
    struct p4_net_msg_hdr nmsg;
    XDR *xdr_enc;
    xdrproc_t xdr_proc;
    char *xdr_buff;
    int xdr_elsize, els_per_buf, xdr_numels;
    int xdr_len, xdr_len1, len_bytes;

    p4_dprintfl(20, "sending msg of type %d from %d to %d via xdr\n",
		type,from,to);

    myid = p4_get_my_id();
    fd = p4_local->conntab[to].port;

    nmsg.msg_type = p4_i_to_n(type);
    nmsg.to = p4_i_to_n(to);
    nmsg.from = p4_i_to_n(from);
    switch (data_type)
    {
      case P4INT:
	xdr_proc = xdr_int;
	xdr_elsize = XDR_INT_LEN;
	break;
      case P4LNG:
	xdr_proc = xdr_long;
	xdr_elsize = XDR_LNG_LEN;
	break;
      case P4FLT:
	xdr_proc = xdr_float;
	xdr_elsize = XDR_FLT_LEN;
	break;
      case P4DBL:
	xdr_proc = xdr_double;
	xdr_elsize = XDR_DBL_LEN;
	break;
      default:
	p4_dprintf("xdr_send: invalid data type %d\n", data_type);
	return (-1);
    }
    xdr_numels = len / xdr_elsize;
    nmsg.msg_len = p4_i_to_n(xdr_numels);
    nmsg.ack_req = p4_i_to_n(ack_req);
    nmsg.data_type = p4_i_to_n(data_type);

    net_send(fd, &nmsg, sizeof(struct p4_net_msg_hdr), FALSE);

    xdr_enc = &(p4_local->xdr_enc);
    xdr_buff = p4_local->xdr_buff;
    els_per_buf = (XDR_BUFF_LEN - XDR_PAD) / xdr_elsize;
    while (xdr_numels > 0)
    {
	if (xdr_numels > els_per_buf)
	    xdr_len = els_per_buf;
	else
	    xdr_len = xdr_numels;
	xdr_len1 = xdr_len;	/* remember xdr_len */
	if (!xdr_setpos(xdr_enc, 0))
	{
	    p4_dprintf("xdr_send: xdr_setpos failed\n");
	    return (-1);
	}
	if (!xdr_array(xdr_enc, &msg, &xdr_len, XDR_BUFF_LEN,
		       xdr_elsize, xdr_proc))
	{
	    p4_dprintf("xdr_send: xdr_array failed\n");
	    return (-1);
	}
	len_bytes = xdr_getpos(xdr_enc);

	flag = (myid < to) ? TRUE : FALSE;
	net_send(fd, xdr_buff, len_bytes, flag);

	nbytes_written += len_bytes;
	xdr_numels -= xdr_len1;
	msg = msg + len_bytes - XDR_PAD;
    }

    if (ack_req & P4_ACK_REQ_MASK)
    {
	wait_for_ack(fd);
    }
    p4_dprintfl(10, "sent msg of type %d from %d to %d via xdr\n",type,from,to);
    return (nbytes_written);
}
#endif

int socket_send(type, from, to, msg, len, data_type, ack_req)
int type, from, to, len, data_type, ack_req;
char *msg;
{
    int fd, flag;
    int sent = 0;
    int done = 0;
    int nleft, rc, n, nfds;
    struct p4_net_msg_hdr nmsg;

    p4_dprintfl(20, "sending msg of type %d from %d to %d via socket\n",type,from,to);
    if (CHECKNODE(to) || CHECKNODE(from))
	p4_error("socket_send: bad header: to/from node is out of range",
		 to * 10000 + from);

    fd = p4_local->conntab[to].port;

    nmsg.msg_type = p4_i_to_n(type);
    nmsg.to = p4_i_to_n(to);
    nmsg.from = p4_i_to_n(from);
    nmsg.msg_len = p4_i_to_n(len);
    nmsg.ack_req = p4_i_to_n(ack_req);
    nmsg.data_type = p4_i_to_n(data_type);

    net_send(fd, &nmsg, sizeof(struct p4_net_msg_hdr), FALSE);
    p4_dprintfl(20, "sent hdr for type %d from %d to %d via socket\n",type,from,to);

    while (sent < len)
    {
	if ((len - sent) > SOCK_BUFF_SIZE)
	    nleft = SOCK_BUFF_SIZE;
	else
	    nleft = len - sent;

	flag = (from < to) ? TRUE : FALSE;
	n = net_send(fd, ((char *) msg) + sent, nleft, flag);

	sent += n;
    }

    if (ack_req & P4_ACK_REQ_MASK)
    {
	wait_for_ack(fd);
    }
    p4_dprintfl(10, "sent msg of type %d from %d to %d via socket\n",type,from,to);
    return (sent);
}


struct p4_msg *socket_recv()
{
    int i, fd, nfds;
    struct p4_msg *tmsg = NULL;
    P4BOOL found = FALSE;
    struct timeval tv;
    fd_set read_fds;

    while (!found)
    {
	tv.tv_sec = 9;
	tv.tv_usec = 0;  /* RMB */
	FD_ZERO(&read_fds);
	for (i = 0; !tmsg && i < p4_global->num_in_proctable; i++)
	{
	    if (p4_local->conntab[i].type == CONN_REMOTE_EST)
	    {
		fd = p4_local->conntab[i].port;
		FD_SET(fd, &read_fds);
	    }
	}
	nfds = select(p4_global->max_connections, &read_fds, 0, 0, &tv);
	if (nfds)
	{
	    for (i = 0; !tmsg && i < p4_global->num_in_proctable; i++)
	    {
		if (p4_local->conntab[i].type == CONN_REMOTE_EST)
		{
		    fd = p4_local->conntab[i].port;
		    if (FD_ISSET(fd,&read_fds)  &&  sock_msg_avail_on_fd(fd))
		    {
			tmsg = socket_recv_on_fd(fd);
			found = TRUE;
			if (tmsg->ack_req & P4_ACK_REQ_MASK)
			{
			    send_ack(fd, tmsg->from);
			}
		    }
		}
	    }
	}
    }
    return (tmsg);
}

struct p4_msg *socket_recv_on_fd(fd)
int fd;
{
    int n, data_type, msg_len;
    struct p4_msg *tmsg;
    struct p4_net_msg_hdr nmsg;

    n = net_recv(fd, &nmsg, sizeof(struct p4_net_msg_hdr));

    data_type = p4_n_to_i(nmsg.data_type);
    if (data_type == P4NOX)
	msg_len = p4_n_to_i(nmsg.msg_len);
    else
    {
	switch (data_type)
	{
	  case P4INT:
	    msg_len = p4_n_to_i(nmsg.msg_len) * XDR_INT_LEN;
	    break;
	  case P4LNG:
	    msg_len = p4_n_to_i(nmsg.msg_len) * XDR_LNG_LEN;
	    break;
	  case P4FLT:
	    msg_len = p4_n_to_i(nmsg.msg_len) * XDR_FLT_LEN;
	    break;
	  case P4DBL:
	    msg_len = p4_n_to_i(nmsg.msg_len) * XDR_DBL_LEN;
	    break;
	  default:
	    p4_error("socket_recv_on_fd: invalid data type %d\n", data_type);
	}
    }

    tmsg = alloc_p4_msg(msg_len);
    tmsg->type = p4_n_to_i(nmsg.msg_type);
    tmsg->from = p4_n_to_i(nmsg.from);
    tmsg->to = p4_n_to_i(nmsg.to);
    tmsg->len = p4_n_to_i(nmsg.msg_len);	/* chgd by xdr_recv below */
    tmsg->data_type = p4_n_to_i(nmsg.data_type);
    tmsg->ack_req = p4_n_to_i(nmsg.ack_req);
    if (tmsg->data_type == P4NOX || p4_local->conntab[tmsg->from].same_data_rep)
    {
	n = net_recv(fd, (char *) &(tmsg->msg), tmsg->len);
    }
    else
    {
#       ifdef CAN_DO_XDR
	n = xdr_recv(fd, tmsg);
#       else
	p4_error("cannot do xdr recvs\n",0);
#       endif
    }
    return (tmsg);
}

P4BOOL socket_msgs_available()
{
    int i, fd;

    for (i = 0; i < p4_global->num_in_proctable; i++)
    {
	if (p4_local->conntab[i].type == CONN_REMOTE_EST)
	{
	    fd = p4_local->conntab[i].port;
	    if (sock_msg_avail_on_fd(fd))
	    {
		return (TRUE);
	    }
	}
    }
    return (FALSE);
}

P4BOOL sock_msg_avail_on_fd(fd)
int fd;
{
    int i, rc, nfds;
    struct timeval tv;
    fd_set read_fds;
    char tempbuf[2];

    rc = FALSE;
    tv.tv_sec = 0;
    tv.tv_usec = 0;
    FD_ZERO(&read_fds);
    FD_SET(fd, &read_fds);
    nfds = select(p4_global->max_connections, &read_fds, 0, 0, &tv);

    if (nfds == -1)
    {
	p4_error("sock_msg_avail_on_fd select", nfds);
    }
    if (nfds)			/* true even for eof */
    {
	/* see if data is on the socket or merely an eof condition */
	/* this should not loop long because the select succeeded */
	while ((rc = recv(fd, tempbuf, 1, MSG_PEEK)) == -1)
	    ;	

	if (rc == 0)	/* if eof */
	{
	    /* eof; a process has closed its socket; may have died */
	    for (i = 0; i < p4_global->num_in_proctable; i++)
		if (p4_local->conntab[i].port == fd)
		    p4_local->conntab[i].type = CONN_REMOTE_DYING;
	}
	else
	    rc = TRUE;
    }
    return (rc);
}

#ifdef CAN_DO_XDR
int xdr_recv(fd, rmsg)
int fd;
struct p4_msg *rmsg;
{
    xdrproc_t xdr_proc;
    XDR *xdr_dec;
    char *xdr_buff, *msg;
    int i, n;
    int msg_len = 0, nbytes_read = 0;
    int xdr_elsize, els_per_buf, xdr_numels;
    int xdr_len, xdr_len1, len_bytes;

    msg = (char *) &(rmsg->msg);

    xdr_dec = &(p4_local->xdr_dec);
    xdr_buff = p4_local->xdr_buff;
    switch (rmsg->data_type)
    {
      case P4INT:
	xdr_proc = xdr_int;
	xdr_elsize = XDR_INT_LEN;
	break;
      case P4LNG:
	xdr_proc = xdr_long;
	xdr_elsize = XDR_LNG_LEN;
	break;
      case P4FLT:
	xdr_proc = xdr_float;
	xdr_elsize = XDR_FLT_LEN;
	break;
      case P4DBL:
	xdr_proc = xdr_double;
	xdr_elsize = XDR_DBL_LEN;
	break;
      default:
	p4_dprintf("xdr_recv: invalid data type %d\n", rmsg->data_type);
	return (-1);
    }
    xdr_numels = rmsg->len;
    els_per_buf = (XDR_BUFF_LEN - XDR_PAD) / xdr_elsize;
    while (xdr_numels > 0)
    {
	if (xdr_numels > els_per_buf)
	    xdr_len = els_per_buf;
	else
	    xdr_len = xdr_numels;
	xdr_len1 = xdr_len;	/* remember xdr_len */

	len_bytes = (xdr_len * xdr_elsize) + XDR_PAD;
    	p4_dprintfl(90, "xdr_recv: reading %d bytes from %d\n", len_bytes, fd);
	n = net_recv(fd, xdr_buff, len_bytes);
	p4_dprintfl(90, "xdr_recv: read %d bytes \n", n);

	if (n < 0)
	{
	    p4_error("xdr_recv net_recv", n);
	}

	if (!xdr_setpos(xdr_dec, 0))
	{
	    p4_dprintf("xdr_recv: xdr_setpos failed\n");
	    return (-1);
	}

	if (!xdr_array(xdr_dec, &msg, &xdr_len, XDR_BUFF_LEN,
		       xdr_elsize, xdr_proc))
	{
	    p4_dprintf("xdr_recv: xdr_array failed\n");
	    return (-1);
	}

	nbytes_read += len_bytes;
	xdr_numels -= xdr_len1;
	msg = msg + len_bytes - XDR_PAD;
	msg_len = msg_len + len_bytes - XDR_PAD;
    }
    rmsg->len = msg_len;
    return (msg_len);
}
#endif

P4VOID wait_for_ack(fd)
int fd;
{
    struct p4_msg *ack;

    p4_dprintfl(30, "waiting for ack \n");
    ack = socket_recv_on_fd(fd);
    while (!(ack->ack_req & P4_ACK_REPLY_MASK))
    {
	queue_p4_message(ack, p4_local->queued_messages);
	ack = socket_recv_on_fd(fd);
    }
    free_p4_msg(ack);
    p4_dprintfl(30, "received ack from %d\n", ack->from);
}

P4VOID send_ack(fd, to)
int fd, to;
{
    struct p4_net_msg_hdr ack;

    p4_dprintfl(30, "sending ack to %d\n", to);
    ack.from = p4_i_to_n(p4_get_my_id());
    ack.data_type = p4_i_to_n(P4NOX);
    ack.msg_len = p4_i_to_n(0);
    ack.to = p4_i_to_n(to);
    ack.ack_req = p4_i_to_n(P4_ACK_REPLY_MASK);
    net_send(fd, &ack, sizeof(ack), FALSE);
    p4_dprintfl(30, "sent ack to %d\n", to);
}

P4VOID shutdown_p4_socks()
/*
  Shutdown all sockets we know about discarding info
  in either direction.
  */
{
    int i;

    if (!p4_local)		/* Need info to be defined */
	return;
    if (!p4_local->conntab)
	return;
    if (p4_local->my_id == LISTENER_ID)
	return;

    for (i = 0; i < p4_num_total_ids(); i++)
	if (p4_local->conntab[i].type == CONN_REMOTE_EST)
	{
	    (P4VOID) shutdown(p4_local->conntab[i].port, 2);
	    (P4VOID) close(p4_local->conntab[i].port);
	}
}
SHAR_EOF
if test -f 'p4_sock_util.c'
then
	echo shar: over-writing existing file "'p4_sock_util.c'"
fi
cat << \SHAR_EOF > 'p4_sock_util.c'
#include "p4.h"
#include "p4_sys.h"
/* p4_net_utils.h generally would suffice here */

extern int errno;
extern char *sys_errlist[];


/*
 *    Utility routines for socket hacking in p4:
 *        P4VOID net_setup_listener(backlog, port, skt)
 *        P4VOID net_setup_anon_listener(backlog, port, skt)
 *        int net_accept(skt)
 *        int net_conn_to_listener(hostname, port)
 *        int net_recv(fd, buf, size)
 *        P4VOID net_send(fd, buf, size)
 *        get_inet_addr(addr)
 *        get_inet_addr_str(str)
 *        dump_sockaddr(who, sa)
 *        dump_sockinfo(msg, fd)
 */

/*
 *    Setup a listener:
 *        get a socket
 *        get a port
 *        listen on the port
 *
 *    Note that this does NOT actually start a listener process, but
 *    merely does the listen command.  It might be executed by a
 *    listener process, but we commonly use it prior to actually
 *    forking off the listener.
 */


P4VOID net_setup_listener(backlog, port, skt)
int backlog;
int port;
int *skt;
{
    struct sockaddr_in sin;
    int optval = TRUE;

    *skt = socket(AF_INET, SOCK_STREAM, 0);
    if (*skt < 0)
	p4_error("net_setup_listener socket", *skt);
#ifdef CAN_DO_SETSOCKOPT
    setsockopt(*skt, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval));
#endif

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = INADDR_ANY;
    sin.sin_port = htons(port);

    if (bind(*skt, (struct sockaddr *) & sin, sizeof(sin)) < 0)
	p4_error("net_setup_listener bind", -1);

    if (listen(*skt, backlog) < 0)
	p4_error("net_setup_listener listen", -1);
}

P4VOID net_setup_anon_listener(backlog, port, skt)
int backlog;
int *port;
int *skt;
{
    int rc, sinlen;
    struct sockaddr_in sin;
    int optval = TRUE;

    *skt = socket(AF_INET, SOCK_STREAM, 0);
    if (*skt < 0)
	p4_error("net_setup_anon_listener socket", *skt);
#ifdef CAN_DO_SETSOCKOPT
    rc = setsockopt(*skt, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval));
#endif

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = INADDR_ANY;
    sin.sin_port = htons(0);

    sinlen = sizeof(sin);

    if (bind(*skt, (struct sockaddr *) & sin, sizeof(sin)) < 0)
	p4_error("net_setup_anon_listener bind", -1);

    if (listen(*skt, backlog) < 0)
	p4_error("net_setup_anon_listener listen", -1);

    getsockname(*skt, (struct sockaddr *) & sin, &sinlen);
    *port = ntohs(sin.sin_port);
}

/*
  Accept a connection on socket skt and return fd of new connection.
  */
int net_accept(skt)
int skt;
{
    struct sockaddr_in from;
    int rc, flags, fromlen, skt2, gotit, sockbuffsize;
    int optval = TRUE;

    fromlen = sizeof(from);
    gotit = 0;
    while (!gotit)
    {
	p4_dprintfl(60, "net_accept - waiting for accept.\n");
	skt2 = accept(skt, (struct sockaddr *) &from, &fromlen);
	p4_dprintfl(60, "net_accept - got accept.\n");
	if (skt2 == -1)
	{
	    if (errno == EINTR)
		continue;
	    else
		p4_error("net_accept accept", skt2);
	}
	else
	    gotit = 1;
    }
#ifdef CAN_DO_SETSOCKOPT
    setsockopt(skt2, IPPROTO_TCP, TCP_NODELAY, (char *) &optval, sizeof(optval));
#endif
    sockbuffsize = SOCK_BUFF_SIZE;
    /******************
      if (setsockopt(skt2,SOL_SOCKET,SO_RCVBUF,(char *)&sockbuffsize,sizeof(sockbuffsize)))
      p4_dprintf("net_accept: setsockopt rcvbuf failed\n");
      if (setsockopt(skt2,SOL_SOCKET,SO_SNDBUF,(char *)&sockbuffsize,sizeof(sockbuffsize)))
      p4_dprintf("net_accept: setsockopt sndbuf failed\n");
      ******************/

    flags = fcntl(skt2, F_GETFL, 0);
    if (flags < 0)
	p4_error("net_accept fcntl1", flags);
    flags |= O_NDELAY;
    fcntl(skt2, F_SETFL, flags);
    if (flags < 0)
	p4_error("net_accept fcntl2", flags);

    return (skt2);
}

int net_conn_to_listener(hostname, port, num_tries)
char *hostname;
int port, num_tries;
{
    int flags, rc, s;
    struct sockaddr_in listener;
    struct hostent *hp;
    P4BOOL optval = TRUE;
    P4BOOL connected = FALSE;

    hp = gethostbyname_p4(hostname);

    /* p4_dprintfl(70, "net_conn_to_listener: host=%s port=%d\n", hostname, port); */
    p4_dprintfl(80, "net_conn_to_listener: host=%s port=%d\n", hostname, port);
    bzero((P4VOID *) &listener, sizeof(listener));
    bcopy((P4VOID *) hp->h_addr, (P4VOID *) &listener.sin_addr, hp->h_length);
    listener.sin_family = hp->h_addrtype;
    listener.sin_port = htons(port);

    connected = FALSE;
    while (!connected && num_tries)
    {
	s = socket(AF_INET, SOCK_STREAM, 0);
	if (s < 0)
	    p4_error("net_conn_to_listener socket", s);

#       ifdef CAN_DO_SETSOCKOPT
	setsockopt(s,IPPROTO_TCP,TCP_NODELAY,(char *) &optval,sizeof(optval));
#       endif

	rc = connect(s, (struct sockaddr *) &listener, sizeof(listener));

	if (rc < 0)
	{
	    close(s);
	    if (--num_tries)
	    {
		p4_dprintfl(00,"net_conn_to_listener: connect to %s failed; will try %d more times \n",hostname,num_tries);
		sleep(1);
	    }
	}
	else
	{
	    connected = TRUE;
	    p4_dprintfl(70,"net_conn_to_listener: connected to %s\n",hostname);
	}
    }
    if (!connected)
	return(-1);

    flags = fcntl(s, F_GETFL, 0);
    if (flags < 0)
	p4_error("net_conn_to_listener fcntl1", flags);
    flags |= O_NDELAY;
    fcntl(s, F_SETFL, flags);
    if (flags < 0)
	p4_error("net_conn_to_listener fcntl2", flags);

    return (s);
}

int net_recv(fd, buf, size)
int fd;
char *buf;
int size;
{
    int recvd = 0;
    int n;

    while (recvd < size)
    {
	n = read(fd, buf + recvd, size - recvd);
	if (n < 0)
	{
	    if (errno == EWOULDBLOCK)
		continue;
	    else
		p4_error("net_recv read", n);
	}
	recvd += n;
    }
    return (recvd);
}

int net_send(fd, buf, size, flag)
int fd;
char *buf;
int size;
int flag;  
/* flag --> fromid < toid; tie-breaker to avoid 2 procs rcving at same time */
/* typically set false for small internal messages, esp. when ids may not */
/*     yet be available */
/* set true for user msgs which may be quite large */
{
    struct p4_msg *dmsg;
    int n, sent = 0;

    while (sent < size)
    {
	n = write(fd, buf + sent, size - sent);
	if (n < 0)
	{
	    if (errno == EWOULDBLOCK)
	    {
		if (flag)
		{
		    if (socket_msgs_available())
		    {
			dmsg = socket_recv();
			queue_p4_message(dmsg, p4_local->queued_messages);
		    }
		}
		continue;
	    }
	    else
	    {
		p4_error("net_send write", n);
	    }
	}
	sent += n;
    }
    return (sent);
}

struct hostent *gethostbyname_p4(hostname)
char *hostname;
{
    struct hostent *hp;
    int i = 100;

    while ((hp = gethostbyname(hostname)) == NULL)
    {
	if (!--i)
	{
	    i = 100;
	    p4_dprintfl(00,"gethostbyname failed 100 times for host %s\n",
			hostname);
	}
    }
    return(hp);
}

P4VOID get_inet_addr(addr)
struct in_addr *addr;
{
    char hostname[100];
    struct hostent *hp;

    hostname[0] = '\0';
    get_qualified_hostname(hostname);
    hp = gethostbyname_p4(hostname);
    bcopy(hp->h_addr, addr, hp->h_length);
}

P4VOID get_inet_addr_str(str)
char *str;
{
    struct in_addr addr;

    get_inet_addr(&addr);
    strcpy(str, (char *) inet_ntoa(addr));
}

#if !defined(CRAY)
/* cray complains about addr being addr of bit field */
/* can probably get around this problem if ever necessary */

P4VOID dump_sockaddr(who,sa)
char *who;
struct sockaddr_in *sa;
{
    unsigned char *addr;

    addr = (unsigned char *) &(sa->sin_addr.s_addr);

    p4_dprintfl(00,"%s: family=%d port=%d addr=%d.%d.%d.%d\n",
		who,
                ntohs(sa->sin_family),
                ntohs(sa->sin_port),
                addr[0], addr[1], addr[2], addr[3]);
}

P4VOID dump_sockinfo(msg, fd)
char *msg;
int fd;
{
    int nl;
    struct sockaddr_in peer, me;

    p4_dprintfl(00, "Dumping sockinfo for fd=%d: %s\n", fd, msg);

    nl = sizeof(peer);
    getpeername(fd, (struct sockaddr *) &peer, &nl);
    dump_sockaddr("Peer",&peer);
	   
    nl = sizeof(me);
    getsockname(fd, (struct sockaddr *) &me, &nl);
    dump_sockaddr("Me", &me);
}

#endif
SHAR_EOF
if test -f 'p4_sock_util.h'
then
	echo shar: over-writing existing file "'p4_sock_util.h'"
fi
cat << \SHAR_EOF > 'p4_sock_util.h'
#ifdef CAN_DO_SOCKET_MSGS
#include <sys/socket.h>
#endif
#include <netdb.h>

#ifdef P4BSD
#include <sys/wait.h>
#include <sys/resource.h>
#endif

#ifdef CAN_DO_SETSOCKOPT
#include <netinet/tcp.h>
#endif

#define NET_RECV_GOOD 0
#define NET_RECV_EOF  -1    

#define NET_EXEC 1
#define NET_DONE 2
#define NET_RESPONSE 3

#define SOCK_BUFF_SIZE 4096

#define UNRESERVED_PORT 5001

struct net_message_t 
{
    int type:32;
    int port:32;
    int success:32;
    char pgm[256];
    char host[128];
    char am_slave[32];
    char message[512];
};

/* Definitions for system messages between slaves */
/* Macros to convert from integer to net byte order and vice versa */
#ifdef P4BSD
#define p4_i_to_n(n)  (int) htonl( (u_long) n)
#define p4_n_to_i(n)  (int) ntohl( (u_long) n)
#endif

#ifdef P4SYSV
#if defined(IPSC860)  &&  !defined(IPSC860_SOCKETS)
#define p4_i_to_n(n)  (n)
#define p4_n_to_i(n)  (n)
#else
#define p4_i_to_n(n)  (int) htonl(n)
#define p4_n_to_i(n)  (int) ntohl(n)
#endif
#endif
SHAR_EOF
if test -f 'p4_utils.c'
then
	echo shar: over-writing existing file "'p4_utils.c'"
fi
cat << \SHAR_EOF > 'p4_utils.c'
#include "p4.h"
#include "p4_sys.h"

char *p4_version()
{
    return(P4_PATCHLEVEL);
}

char *p4_machine_type()
{
    return(P4_MACHINE_TYPE);
}


#if defined(IPSC860)
#define MYNODE mynode
#endif
#if defined(CM5)
#define MYNODE CMMD_self_address
#endif

int p4_initenv(argc, argv)
int *argc;
char **argv;
{
    int i, rc;
    P4BOOL am_slave = FALSE;

    sprintf(whoami, "xm_%d", getpid());

    for (i = 0; i < *argc; i++)
	if (strcmp(argv[i], "-amp4slave") == 0)
	    am_slave = TRUE;

#   if defined(CM5)
    CMMD_fset_io_mode( stdin,  CMMD_independent );
    CMMD_fset_io_mode( stdout, CMMD_independent );
    CMMD_fset_io_mode( stderr, CMMD_independent );
    if (MYNODE() != 0)
        am_slave = TRUE;
#   endif

    p4_local  = NULL;
    p4_global = NULL;

    if (am_slave)
    {
#       if defined(IPSC860)  ||  defined(CM5)
        if (MYNODE() == 0)
	{
#           if defined(IPSC860_SOCKETS)  ||  defined(CM5_SOCKETS)
	    rc = rm_start(argc, argv);
#           endif
	}
	else
	{
	    rc = ns_start(argc, argv);
	}
#       else
	rc = rm_start(argc, argv);
#       endif
	ALOG_SETUP(p4_local->my_id,ALOG_TRUNCATE);
    }
    else
    {
	rc = bm_start(argc, argv);
        ALOG_MASTER(0,ALOG_TRUNCATE);
        ALOG_DEFINE(BEGIN_USER,"beg_user","");
        ALOG_DEFINE(END_USER,"end_user","");
        ALOG_DEFINE(BEGIN_SEND,"beg_send","");
        ALOG_DEFINE(END_SEND,"end_send","");
        ALOG_DEFINE(BEGIN_RECV,"beg_recv","");
        ALOG_DEFINE(END_RECV,"end_recv","");
        ALOG_DEFINE(BEGIN_WAIT,"beg_wait","");
        ALOG_DEFINE(END_WAIT,"end_wait","");
    }
    ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");
    return (rc);
}

char *p4_shmalloc(n)
int n;
{
    char *rc;

    if ((rc = MD_shmalloc(n)) == NULL)
	p4_dprintf("p4_shmalloc returning NULL; request = %d bytes\n",n);
    return (rc);
}

P4VOID p4_shfree(p)
char *p;
{
    MD_shfree(p);
}

int p4_num_cluster_ids()
{
    return (p4_global->local_slave_count + 1);
}

int p4_num_total_ids()
{
    return (p4_global->num_in_proctable);
}

int p4_num_total_slaves()
{
    return (p4_global->num_in_proctable - 1);
}


P4VOID p4_global_barrier(type)
int type;
{
    double dummy[1];

    p4_global_op(type, (char *) dummy, 1, sizeof(double), p4_dbl_sum_op, P4DBL);
}


P4VOID p4_get_cluster_masters(numids, ids)
int *numids, ids[];
{
    int node;

    ids[0] = 0;
    *numids = 1;
    for (node = 1; node < p4_global->num_in_proctable; node++)
    {
	if (p4_global->proctable[node].slave_idx != 0)
	    continue;
	ids[(*numids)++] = node;
    }
}


P4VOID p4_get_cluster_ids(start, end)
int *start;
int *end;
{

    *start = p4_global->low_cluster_id;
    *end = p4_global->hi_cluster_id;
}

/* This is used to figure out the local id of the calling process by
 * indexing into the proctable until you find a hostname and a unix id
 * that are the same as yours.
 */
int p4_get_my_id_from_proc()
{
    int i, my_unix_id;
    struct proc_info *pi;
    struct hostent *myhp, *pghp;

#   if (defined(IPSC860)  &&  !defined(IPSC860_SOCKETS))  ||  \
       (defined(CM5)      &&  !defined(CM5_SOCKETS))
    return(MYNODE());
#   else
    my_unix_id = getpid();
    if (p4_local->my_id == LISTENER_ID)
	return (LISTENER_ID);
    myhp = gethostbyname_p4(p4_global->my_host_name);

    for (pi = p4_global->proctable, i = 0; i < p4_global->num_in_proctable; i++, pi++)
    {
	if (pi->unix_id == my_unix_id)
	{
	    if (strcmp(pi->host_name, p4_global->my_host_name) == 0)
	    {
		return (i);
	    }
	    else
	    {
		pghp = gethostbyname_p4(pi->host_name);
		if (bcmp(myhp->h_addr, pghp->h_addr, myhp->h_length) == 0)
		{
		    return (i);
		}
	    }
	}
    }
    p4_dprintf("process not in process table; my_unix_id = %d my_host=%s\n",
	       getpid(), p4_global->my_host_name);
    p4_dprintf("Probable cause:  local slave on uniprocessor without shared memory\n");
    p4_dprintf("Probable fix:  ensure only one process on %s\n",p4_global->my_host_name);
    p4_dprintf("(on master process this means 'local 0' in the procgroup file)\n");
    p4_dprintf("You can also remake p4 with SYSV_IPC set in the OPTIONS file\n");
    p4_error("p4_get_my_id_from_proc",0);
#   endif
}

int p4_get_my_id()
{
    return (p4_local->my_id);
}

int p4_get_my_cluster_id()
{
#   if (defined(IPSC860)  &&  !defined(IPSC860_SOCKETS))  ||  \
       (defined(CM5)      &&  !defined(CM5_SOCKETS))
    return(MYNODE());
#   else
    if (p4_local->my_id == LISTENER_ID)
	return (LISTENER_ID);
    else
	return (p4_global->proctable[p4_local->my_id].slave_idx);
#   endif
}

P4BOOL p4_am_i_cluster_master()
{
    if (p4_local->my_id == LISTENER_ID)
	return (0);
    else
	return (p4_global->proctable[p4_local->my_id].slave_idx == 0);
}

P4BOOL in_same_cluster(i, j)
int i, j;
{
    return (p4_global->proctable[i].group_id ==
	    p4_global->proctable[j].group_id);
}

P4VOID p4_cluster_shmem_sync(cluster_shmem)
P4VOID **cluster_shmem;
{
    int myid = p4_get_my_cluster_id();

    if (myid == 0)  /* cluster master */
	p4_global->cluster_shmem = *cluster_shmem;
    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());
    if (myid != 0)
	*cluster_shmem = p4_global->cluster_shmem;
}

#if defined(USE_XX_SHMALLOC)
/* This is not machine dependent code but is only used on some machines */

/*
  Memory management routines from ANSI K&R C, modified to manage
  a single block of shared memory.
  Have stripped out all the usage monitoring to keep it simple.

  To initialize a piece of shared memory:
    xx_init_shmalloc(char *memory, unsigned nbytes)

  Then call xx_shmalloc() and xx_shfree() as usual.
*/

#define LOG_ALIGN 4
#define ALIGNMENT (1 << LOG_ALIGN)

/* ALIGNMENT is assumed below to be bigger than sizeof(p4_lock_t) +
   sizeof(Header *), so do not reduce LOG_ALIGN below 4 */

union header
{
    struct
    {
	union header *ptr;	/* next block if on free list */
	unsigned size;		/* size of this block */
    } s;
    char align[ALIGNMENT];	/* Align to ALIGNMENT byte boundary */
};

typedef union header Header;

static Header **freep;		/* pointer to pointer to start of free list */
static p4_lock_t *shmem_lock;	/* Pointer to lock */

P4VOID xx_init_shmalloc(memory, nbytes)
char *memory;
unsigned nbytes;
/*
  memory points to a region of shared memory nbytes long.
  initialize the data structures needed to manage this memory
*/
{
    int nunits = nbytes >> LOG_ALIGN;
    Header *region = (Header *) memory;

    /* Quick check that things are OK */

    if (ALIGNMENT != sizeof(Header) ||
	ALIGNMENT < (sizeof(Header *) + sizeof(p4_lock_t)))
	p4_error("xx_init_shmem: Alignment is wrong", ALIGNMENT);

    if (!region)
	p4_error("xx_init_shmem: Passed null pointer", 0);

    if (nunits < 2)
	p4_error("xx_init_shmem: Initial region is ridiculously small",
		 (int) nbytes);

    /*
     * Shared memory region is structured as follows
     * 
     * 1) (Header *) freep ... free list pointer 2) (p4_lock_t) shmem_lock ...
     * space to hold lock 3) padding up to alignment boundary 4) First header
     * of free list
     */

    freep = (Header **) region;	/* Free space pointer in first block  */
    shmem_lock = (p4_lock_t *) (freep + 1);	/* Lock still in first block */
    (region + 1)->s.ptr = *freep = region + 1;	/* Data in rest */
    (region + 1)->s.size = nunits - 1;	/* One header consumed already */

#   ifdef SYSV_IPC
    shmem_lock->semid = sysv_semid0;
    shmem_lock->semnum = 0;
#   else
    p4_lock_init(shmem_lock);                /* Initialize the lock */
#   endif

}

char *xx_shmalloc(nbytes)
unsigned nbytes;
{
    Header *p, *prevp;
    char *address = (char *) NULL;
    unsigned nunits;

    /* Force entire routine to be single threaded */
    (P4VOID) p4_lock(shmem_lock);

    nunits = ((nbytes + sizeof(Header) - 1) >> LOG_ALIGN) + 1;

    prevp = *freep;
    for (p = prevp->s.ptr;; prevp = p, p = p->s.ptr)
    {
	if (p->s.size >= nunits)
	{			/* Big enuf */
	    if (p->s.size == nunits)	/* exact fit */
		prevp->s.ptr = p->s.ptr;
	    else
	    {			/* allocate tail end */
		p->s.size -= nunits;
		p += p->s.size;
		p->s.size = nunits;
	    }
	    *freep = prevp;
	    address = (char *) (p + 1);
	    break;
	}
	if (p == *freep)
	{			/* wrapped around the free list ... no fit
				 * found */
	    address = (char *) NULL;
	    break;
	}
    }

    /* End critical region */
    (P4VOID) p4_unlock(shmem_lock);

    if (address == NULL)
	p4_dprintf("xx_shmalloc: returning NULL; requested %d bytes\n",nbytes);
    return address;
}

P4VOID xx_shfree(ap)
char *ap;
{
    Header *bp, *p;

    /* Begin critical region */
    (P4VOID) p4_lock(shmem_lock);

    if (!ap)
	return;			/* Do nothing with NULL pointers */

    bp = (Header *) ap - 1;	/* Point to block header */

    for (p = *freep; !(bp > p && bp < p->s.ptr); p = p->s.ptr)
	if (p >= p->s.ptr && (bp > p || bp < p->s.ptr))
	    break;		/* Freed block at start of end of arena */

    if (bp + bp->s.size == p->s.ptr)
    {				/* join to upper neighbour */
	bp->s.size += p->s.ptr->s.size;
	bp->s.ptr = p->s.ptr->s.ptr;
    }
    else
	bp->s.ptr = p->s.ptr;

    if (p + p->s.size == bp)
    {				/* Join to lower neighbour */
	p->s.size += bp->s.size;
	p->s.ptr = bp->s.ptr;
    }
    else
	p->s.ptr = bp;

    *freep = p;

    /* End critical region */
    (P4VOID) p4_unlock(shmem_lock);
}
#endif

P4VOID get_pipe(end_1, end_2)
int *end_1;
int *end_2;
{
    int p[2];

#   if defined(IPSC860)  ||  defined(CM5)
    p4_dprintf("WARNING: get_pipe: socketpair assumed unavailable on this machine\n");
    return;
#   else
    if (socketpair(AF_UNIX, SOCK_STREAM, 0, p) < 0)
	p4_error("get_pipe: socketpair failed ", -1);
    *end_1 = p[0];
    *end_2 = p[1];
#   endif
}

P4VOID setup_conntab()
{
    int i, my_id;

    p4_dprintfl(60, "setup_conntab: myid=%d, switch_port=%d, app_id=%s\n",
		p4_local->my_id,
		p4_global->proctable[p4_local->my_id].switch_port,
		p4_global->application_id);
    p4_local->conntab = (struct connection *)
	p4_malloc(p4_global->num_in_proctable * sizeof(struct connection));
    my_id = p4_get_my_id();

    for (i = 0; i < p4_global->num_in_proctable; i++)
    {
	if (i == my_id)
	{
	    p4_local->conntab[i].type = CONN_ME;
	    p4_local->conntab[i].port = 0;
	}
	else if (in_same_cluster(i, my_id))
	{
	    p4_local->conntab[i].type = CONN_LOCAL;
#           if defined(IPSC860) || defined(CM5)
	    p4_local->conntab[i].port = MYNODE() + i - p4_local->my_id;
#           endif
#           if defined(TCMP)
	    p4_local->conntab[i].port = i - p4_global->low_cluster_id;
#           endif
	}
	else if ((p4_global->proctable[my_id].switch_port != -1) &&
		 (p4_global->proctable[i].switch_port != -1) &&
		 (p4_global->proctable[my_id].switch_port !=
		  p4_global->proctable[i].switch_port))
	{
	    p4_local->conntab[i].type = CONN_REMOTE_SWITCH;
	    p4_local->conntab[i].switch_port = p4_global->proctable[i].switch_port;
	}
	else
	{
	    p4_local->conntab[i].type = CONN_REMOTE_NON_EST;
	    p4_local->conntab[i].port = p4_global->proctable[i].port;
	}
    }
    p4_dprintfl(60, "conntab after setup_conntab:\n");
    dump_conntab(60);
}

#ifdef SYSV_IPC
P4VOID remove_sysv_ipc()
{
    int i;
    struct p4_global_data *g = p4_global;

    /* ignore -1 return codes below due to multiple processes cleaning
       up the same sysv stuff; commented out "if" used to make sure
       that only the cluster master cleaned up in each cluster
    */
    /* if (p4_local != NULL  &&  p4_get_my_cluster_id() != 0) return; */

    semctl(g->sysv_semid[0],0,IPC_RMID,0);  /* delete initial set */
    if (sysv_shmid[0] == -1)
	return;
    for (i=0; i < sysv_num_shmids; i++)
        shmctl(sysv_shmid[i],IPC_RMID,0);
    if (g == NULL)
        return;
    for (i=1; i < g->sysv_num_semids; i++)  /* delete other sets */
    {
	semctl(g->sysv_semid[i],0,IPC_RMID,0);
    }
}
#endif

int p4_wait_for_end()
{
    int status;
    int i, n_forked_slaves, pid;
    struct slave_listener_msg msg;

    ALOG_LOG(p4_local->my_id,END_USER,0,"");
    ALOG_OUTPUT;

    /* Wait for all forked processes except listener to die */
    p4_dprintfl(90, "enter wait_for_end nfpid=%d\n",p4_global->n_forked_pids);
    if (p4_local->listener_fd == (-1))
        n_forked_slaves = p4_global->n_forked_pids;
    else
        n_forked_slaves = p4_global->n_forked_pids - 1;
    for (i = 0; i < n_forked_slaves; i++)
    {
	pid = wait(&status);
	p4_dprintfl(90, "detected that proc %d died \n", pid);
    }

#   ifdef CAN_DO_SOCKET_MSGS
    /* Tell the listener to die and wait for him to do so */
    if (p4_local->listener_fd != (-1))
    {
	p4_dprintfl(90, "tell listener to die listpid=%d fd=%d\n",
		    p4_global->listener_pid, p4_local->listener_fd);
	msg.type = p4_i_to_n(DIE);
	msg.from = p4_i_to_n(p4_get_my_id());
	net_send(p4_local->listener_fd, &msg, sizeof(msg), FALSE);
	pid = wait(&status);
	p4_dprintfl(90, "detected that proc %d died \n", pid);
    }
#   endif

#   ifdef SYSV_IPC
    remove_sysv_ipc();
#   endif

    if (p4_get_my_id())
        p4_dprintfl(20,"process exiting\n");
    p4_dprintfl(90, "exit wait_for_end \n");
    return (0);
}


/* static variables private to fork_p4 and zap_p4_processes */
static int n_pids = 0;
static int pid_list[P4_MAXPROCS];

int fork_p4()
/*
  Wrapper round fork for sole purpose of keeping track of pids so 
  that can signal error conditions.  See zap_p4_processes.
*/
{
    int pid;

#   if defined(IPSC860)  ||  defined(CM5)
    p4_error("p4_fork: nodes cannot fork processes",0);    
#   else
    if (p4_global->n_forked_pids >= P4_MAXPROCS)
	p4_error("forking too many local processes; max = ", P4_MAXPROCS);
    p4_global->n_forked_pids++;

    fflush(stdout);
    pid = fork();

    if (pid > 0)
    {
	/* Parent process */
	pid_list[n_pids++] = pid;
    }
    else if (pid == 0)
    {
	/* Child process */
	pid_list[n_pids++] = getppid();
    }
    else
	p4_error("fork_p4: fork failed", pid);
#   endif

    return pid;
}

P4VOID zap_p4_processes()
{
    int n;
    
    if (p4_global == NULL)
        return;
    n = p4_global->n_forked_pids;
    while (n--)
    {
	kill(pid_list[n], SIGINT);
    }
}

P4VOID get_qualified_hostname(str)
char *str;
{
#   if (defined(IPSC860)  &&  !defined(IPSC860_SOCKETS))  ||  \
       (defined(CM5)      &&  !defined(CM5_SOCKETS))
    strcpy(str,"cube_node");
#else
    if (*str == '\0')
        gethostname(str, 100);
    if (*local_domain != '\0'  &&  !index(str,'.'))
    {
	strcat(str,".");
	strcat(str,local_domain);
    }
#endif
}


int getswport(hostname)
char *hostname;
{
    char local_host[256];

#ifdef CAN_DO_SWITCH_MSGS
    if (strcmp(hostname, "local") == 0)
    {
	local_host[0] = '\0';
	get_qualified_hostname(local_host);
	return getswport(local_host);
    }
    if (strcmp(hostname, "hurley") == 0)
	return 1;
    if (strcmp(hostname, "hurley.tcg.anl.gov") == 0)
	return 1;
    if (strcmp(hostname, "hurley.mcs.anl.gov") == 0)
	return 1;
    if (strcmp(hostname, "campus.mcs.anl.gov") == 0)
	return 2;
    if (strcmp(hostname,"mpp1") == 0)
      return 3;
    if (strcmp(hostname,"mpp2") == 0)
      return 28;
    if (strcmp(hostname,"mpp3") == 0)
      return 6;
    if (strcmp(hostname,"mpp4") == 0)
      return 7;
    if (strcmp(hostname,"mpp7") == 0)
      return 14;
    if (strcmp(hostname,"mpp8") == 0)
      return 25;
    if (strcmp(hostname,"mpp9") == 0)
      return 20;
    if (strcmp(hostname,"mpp10") == 0)
      return 11;
#endif

    return -1;
}

P4BOOL same_data_representation(id1,id2)
{
    struct proc_info *p1 = &(p4_global->proctable[id1]);
    struct proc_info *p2 = &(p4_global->proctable[id2]);

    return (data_representation(p1->machine_type) == data_representation(p2->machine_type));
}
SHAR_EOF
if test -f 'p4_sys.h'
then
	echo shar: over-writing existing file "'p4_sys.h'"
fi
cat << \SHAR_EOF > 'p4_sys.h'
#ifdef P4BSD
#include <strings.h>
#endif
#ifdef P4SYSV
#include <string.h>
#endif

#include "p4_patchlevel.h"
#include "p4_sock_util.h"    

#ifndef TRUE
#define TRUE  1
#define FALSE 0
#endif

extern int errno;
extern char *sys_errlist[];

#define PRECV_EOF -1

#define LISTENER_ATTN_SIGNAL SIGUSR1

#define LISTENER_ID (-99)

#include "p4_sys_funcs.h"

#include "p4_defs.h"
#include "p4_macros.h"

#include "p4_globals.h"


#define BEGIN_USER      101
#define END_USER        102
#define BEGIN_SEND      103
#define END_SEND        104
#define BEGIN_RECV      105
#define END_RECV        106
#define BEGIN_WAIT      107
#define END_WAIT        108

#define UPDATE_NUM_SUBPROBS     200
#define REQUEST_MONITOR_ENTRY   201
#define ENTER_MONITOR           202
#define EXIT_MONITOR            203
#define OPEN_DOOR               204
#define ENTER_DELAY_QUEUE       205
#define EXIT_DELAY_QUEUE        206
#define SECRET_EXIT_MONITOR     207
#define PBDONE                  208
#define PGDONE                  209

SHAR_EOF
if test -f 'p4_sys_funcs.h'
then
	echo shar: over-writing existing file "'p4_sys_funcs.h'"
fi
cat << \SHAR_EOF > 'p4_sys_funcs.h'
char *xx_malloc();
char *MD_shmalloc();
char *print_conn_type();
char *xx_shmalloc();
int MD_clock();
P4VOID get_qualified_hostname();
P4VOID MD_initenv();
P4VOID MD_initmem();
P4VOID MD_malloc_hint();
P4VOID MD_shfree();
P4VOID MD_start_cube_slaves();
int bm_start()	;
P4VOID compute_conntab();
P4VOID request_connection();
int create_bm_processes();
P4VOID create_one_remote()	;
int create_remote_processes();
P4VOID create_rm_processes()	;
P4VOID dump_conntab();
P4VOID dump_global()	;
P4VOID dump_listener()	;
P4VOID dump_local()	;
P4VOID dump_procgroup()	;
P4VOID dump_tmsg();
int establish_connection();
P4VOID exec_pgm();
int fork_p4();
P4VOID free_p4_msg();
P4VOID free_quel();
P4VOID get_inet_addr();
P4VOID get_inet_addr_str();
P4VOID dump_sockaddr();
P4VOID dump_sockinfo();
P4VOID get_pipe()	;
int getswport();
P4VOID handle_connection_interrupt();
P4BOOL MD_i860_msgs_available();
P4BOOL MD_CM5_msgs_available();
int MD_i860_send();
int MD_CM5_send();
struct p4_msg *MD_i860_recv();
struct p4_msg *MD_CM5_recv();
P4BOOL in_same_cluster();
P4VOID init_avail_buffs();
P4VOID initialize_msg_queue()	;
int install_in_proctable();
P4VOID kill_server();
P4VOID listener();
struct listener_data *alloc_listener_info();
struct local_data *alloc_local_bm();
struct local_data *alloc_local_listener();
struct local_data *alloc_local_rm();
struct local_data *alloc_local_slave();
int myhost();
int net_accept()	;
int net_conn_to_addr_listener()	;
int net_conn_to_listener()	;
int net_conn_to_named_listener()	;
int net_create_slave()	;
int net_recv()	;
int net_send()	;
P4VOID net_setup_anon_listener()	;
P4VOID net_setup_listener()	;
P4VOID net_setup_named_listener()	;
int num_in_mon_queue();
P4VOID alloc_global();
struct p4_msg *alloc_p4_msg();
struct p4_msg *get_tmsg();
struct p4_msg *recv_message();
struct p4_msg *search_p4_queue();
struct p4_msg *shmem_recv();
struct p4_msg *socket_recv();
struct p4_msg *socket_recv_on_fd();
struct p4_queued_msg *alloc_quel();
P4VOID process_args()	;
P4BOOL process_connect_request()	;
int process_connection();
P4BOOL process_slave_message()	;
struct p4_procgroup *alloc_procgroup();
struct p4_procgroup *read_procgroup()	;
P4VOID procgroup_to_proctable();
P4VOID queue_p4_message();
P4VOID reaper();
P4VOID receive_proc_table()	;
int rm_newline();
int rm_start();
P4VOID send_ack()	;
int send_message();
P4VOID sync_with_remotes();
P4VOID send_proc_table();
P4VOID setup_conntab();
P4BOOL shmem_msgs_available();
int shmem_send();
P4VOID shutdown_p4_socks();
int simple_lock();
int simple_unlock();
P4BOOL sock_msg_avail_on_fd();
P4BOOL socket_msgs_available();
int socket_send();
int start_slave();
int subtree_broadcast_p4();
P4VOID trap_sig_errs();
P4VOID wait_for_ack()	;
P4VOID waitspin();
int xdr_recv();
int xdr_send();
int data_representation();
P4BOOL same_data_representation();
P4VOID xx_init_shmalloc();
P4VOID xx_shfree();
P4VOID zap_p4_processes();
P4BOOL MD_tcmp_msgs_available();
struct p4_msg *MD_tcmp_recv();
int MD_tcmp_send();
struct hostent *gethostbyname_p4();

#ifdef SYSV_IPC
int init_sysv_semset();
P4VOID MD_lock_init();
P4VOID MD_lock();
P4VOID MD_unlock();
P4VOID remove_sysv_ipc();
#endif
SHAR_EOF
if test -f 'p4_rm.c'
then
	echo shar: over-writing existing file "'p4_rm.c'"
fi
cat << \SHAR_EOF > 'p4_rm.c'
#include "p4.h"
#include "p4_sys.h"

/*
  Defining REDIRECT_OUTPUT allows you to redirect output from
  processes as follows.
  Defining OUT_TO_TERM tty allows you to redirect the
  output from various processes to separate windows.
  */

#if defined(REDIRECT_OUTPUT)
#if defined(OUT_TO_TERM)
#define P4_OUTFILE "/dev/ttyp4"
#else
#define P4_OUTFILE "/u/rbutler/p4/tests/out"
#endif
#endif

static int rm_num;

int rm_start(argc, argv)
int *argc;
char **argv;
{
    int bm_fd, bm_port;
    char *s,*bm_host;
    extern char whoami[];
    struct net_initial_handshake hs;
    struct slave_listener_msg lmsg;
    struct bm_rm_msg msg;
    int type, rc, i, numslaves;
    struct p4_global_data *g;
    char outfile[128];

    trap_sig_errs(); 		/* Errors can happen any time */

    sprintf(whoami, "rm_%d", getpid());
    p4_dprintfl(20,"remote master starting\n");

#if defined(REDIRECT_OUTPUT)
    freopen(P4_OUTFILE, "w", stdout);
    freopen(P4_OUTFILE, "w", stderr);
#endif

    if (*argc < 4)
	p4_error("Invalid arguments to remote_master", *argc);

    bm_host = argv[1];
    bm_port = atoi(argv[2]);

    bm_fd = net_conn_to_listener(bm_host, bm_port, 5);
    if (bm_fd < 0)
	p4_error("rm_start: net_conn_to_listener failed", bm_port);

    net_recv(bm_fd, &hs, sizeof(hs));
    hs.pid = (int) htonl(getpid());
    net_send(bm_fd, &hs, sizeof(hs), FALSE);

#   ifdef SYSV_IPC
    sysv_num_shmids = 0;
    sysv_shmid[0]  = -1;
    sysv_semid0 = init_sysv_semset(0);
#   endif

    /* Get the initialization information from the bm */

    rc = net_recv(bm_fd, &msg, sizeof(msg));
    if (rc == PRECV_EOF)
	p4_error("rm_start: got EOF on net_recv", bm_fd);
    type = p4_n_to_i(msg.type);
    if (type != INITIAL_INFO)
	p4_error("rm_start: unknown type, expecting INITIAL_INFO, type=", type);
    if (strcmp(msg.version,p4_version()) != 0)
    {
	p4_dprintf("my version is %s\n",p4_version());
	p4_error("version does not match master \n",0);
    }

    if ((s = (char *) rindex(msg.pgm,'/'))  ==  NULL)
    {
	p4_dprintf("my fullpathname is invalid: %s\n",msg.pgm);
	p4_error("fullpathname is invalid \n",0);
    }
    else
    {
	*s = '\0';  /* chg to directory name only */
	chdir(msg.pgm);
    }

    globmemsize = p4_n_to_i(msg.memsize);
    logging_flag = p4_n_to_i(msg.logging_flag);
    if (logging_flag)
	ALOG_ENABLE;
    else
	ALOG_DISABLE;

    MD_initmem(globmemsize);
    alloc_global();  /* sets p4_global */
    g = p4_global;
    p4_local = alloc_local_rm();
    g->local_communication_only = FALSE;
    g->num_in_proctable = p4_n_to_i(msg.numinproctab);
    numslaves = p4_n_to_i(msg.numslaves);
    rm_num = p4_n_to_i(msg.rm_num);
    debug_level = p4_n_to_i(msg.debug_level);
    strcpy(outfile, msg.outfile);
    strcpy(p4_global->application_id, msg.application_id);
    p4_dprintfl(90, "got numslaves=%d outfile=%s rm_num=%d dbglvl=%d appid=%s\n",
		numslaves, outfile, rm_num, debug_level, msg.application_id);

    MD_initenv();
    usc_init();

    if (*outfile)
    {
	freopen(outfile, "w", stdout);
	freopen(outfile, "w", stderr);
    }

    SIGNAL_P4(LISTENER_ATTN_SIGNAL, handle_connection_interrupt);
    p4_lock(&g->slave_lock);
    create_rm_processes(numslaves, bm_fd);

    /* Grab the whole proc table from the bm */
    p4_dprintfl(90, "receiving proc table\n");
    receive_proc_table(bm_fd);

    /* let local slaves use proctable to identify themselves */
    p4_unlock(&g->slave_lock);

    sprintf(whoami, "rm_%d_%d", rm_num, getpid());
    p4_local->my_id = p4_get_my_id_from_proc();

    p4_global->low_cluster_id = 
	p4_local->my_id - p4_global->proctable[p4_local->my_id].slave_idx;
    p4_global->hi_cluster_id = 
	p4_global->low_cluster_id + p4_global->local_slave_count;

    setup_conntab();

    if (p4_local->conntab[0].type == CONN_REMOTE_SWITCH)
    {
	p4_local->conntab[0].switch_port = p4_global->proctable[0].switch_port;
	p4_local->conntab[0].port = bm_fd;
    }
    else if (p4_local->conntab[0].type == CONN_REMOTE_NON_EST)
    {
	p4_local->conntab[0].type = CONN_REMOTE_EST;
	p4_local->conntab[0].port = bm_fd;
	p4_local->conntab[0].same_data_rep =
	    same_data_representation(p4_local->my_id,0);
    }
    else
    {
	p4_error("rm_start: invalid conn type in conntab ",
		 p4_local->conntab[0].type);
    }

    sprintf(whoami, "p%d_%d", p4_get_my_id(), getpid());


#if defined(IPSC860)  ||  defined(CM5)
    for (i = 1; i < numslaves; i++)
    {
#       if defined(IPSC860)
	csend((long) INITIAL_INFO, &msg, (long) sizeof(struct bm_rm_msg), 
              (long) i, (long) NODE_PID);
	csend((long) INITIAL_INFO, p4_global->proctable, 
              (long) sizeof(p4_global->proctable), (long) i, (long) NODE_PID);
#       endif
#       if defined(CM5)
	CMMD_send_noblock(i, INITIAL_INFO,  &msg,sizeof(struct bm_rm_msg));
	CMMD_send_noblock(i, INITIAL_INFO,  p4_global->proctable, 
                          sizeof(p4_global->proctable));
#       endif
    }
#endif

    /* 
       sync with local slaves thus insuring that they have the proctable before 
       syncing with bm (this keeps bm and its slaves  from interrupting the local 
       processes too early; then re-sync with local slaves (thus permitting them to 
       interrupt remotes)
    */
    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());
    
    msg.type = p4_i_to_n(SYNC_MSG);
    net_send(bm_fd, &msg, sizeof(msg), FALSE);
    msg.type = -1;  /* reset to verify type received next */
    rc = net_recv(bm_fd, &msg, sizeof(msg));
    type = p4_n_to_i(msg.type);
    if (type != SYNC_MSG)
	p4_error("rm_start: unknown type, expecting SYNC_MSG, type=", type);
    
    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());

    return(0);
}


P4VOID create_rm_processes(nslaves, bm_fd)
int nslaves;
int bm_fd;
{
    struct p4_global_data *g = p4_global;
    struct listener_data *l;
    int end_1, end_2, slave_pid, listener_pid;
    int slave_idx, listener_port, listener_fd;
    char rm_host[100];
    int i, rm_switch_port;
    struct bm_rm_msg bm_msg;

#   if !defined(IPSC860)  &&  !defined(CM5)
    if (nslaves > P4_MAX_MSG_QUEUES)
	p4_error("create_rm_processes: more slaves than msg queues \n", nslaves);
#   endif

    /*
     * Allocate the listener's local data area, since this process will
     * eventually become the listener.
     */

    l = listener_info = alloc_listener_info();

    net_setup_anon_listener(10, &listener_port, &listener_fd);

    p4_dprintfl(70, "created listener on port %d fd %d\n", listener_port,
		listener_fd);

    /* Send off the listener info to the bm */
    bm_msg.type = p4_i_to_n(REMOTE_LISTENER_INFO);
    bm_msg.port = p4_i_to_n(listener_port);
    net_send(bm_fd, &bm_msg, sizeof(struct bm_rm_msg), FALSE);

    rm_host[0] = '\0';
    get_qualified_hostname(rm_host);
    rm_switch_port = getswport(rm_host);

    /* Send my info to the bm */
    bm_msg.type = p4_i_to_n(REMOTE_MASTER_INFO);
    bm_msg.slave_idx = p4_i_to_n(0);
    bm_msg.slave_pid = p4_i_to_n(getpid());
    bm_msg.switch_port = p4_i_to_n(rm_switch_port);
    strcpy(bm_msg.host_name,rm_host);
    strcpy(bm_msg.machine_type,P4_MACHINE_TYPE);
    net_send(bm_fd, &bm_msg, sizeof(struct bm_rm_msg), FALSE);

    g->local_slave_count = 0;

#   ifdef TCMP
    tcmp_init(NULL,p4_get_my_cluster_id(),shmem_getclunid());
#   endif

#   if defined(IPSC860)  ||  defined(CM5)
    for (slave_idx = 1; slave_idx <= nslaves - 1; slave_idx++)
    {
#       if defined(IPSC860)
	crecv(INITIAL_INFO, &bm_msg, (long) sizeof(struct bm_rm_msg));
#       endif
#       if defined(CM5)
        CMMD_receive(CMMD_ANY_NODE, INITIAL_INFO, (void *) &bm_msg, 
                     sizeof(struct bm_rm_msg));
#       endif
	net_send(bm_fd, &bm_msg, sizeof(struct bm_rm_msg), FALSE);
	g->local_slave_count++;
    }
#   else

    get_pipe(&end_1, &end_2);
    for (slave_idx = 1; slave_idx <= nslaves - 1; slave_idx++)
    {
	p4_dprintfl(20,"remote master creating local slave %d\n",slave_idx);
		    
	slave_pid = fork_p4();
	if (slave_pid)
	    p4_dprintfl(10,"remote master created local slave %d\n",slave_idx);
	if (slave_pid == 0)
	{
	    /* In the slave process */

	    sprintf(whoami, "rm_s_%d_%d_%d", rm_num, slave_idx, getpid());

	    close(listener_fd);
	    p4_local = alloc_local_slave();
	    p4_local->listener_fd = end_1;
	    close(end_2);

            SIGNAL_P4(LISTENER_ATTN_SIGNAL, handle_connection_interrupt);


	    /* hang for a valid proctable */
	    p4_lock(&g->slave_lock);
	    p4_unlock(&g->slave_lock);

	    p4_local->my_id = p4_get_my_id_from_proc();
	    sprintf(whoami, "p%d_%d", p4_get_my_id(), getpid());
	    setup_conntab();
	    usc_init();

#           ifdef TCMP
            tcmp_init(NULL,p4_get_my_cluster_id(),shmem_getclunid());
#           endif

	    /* 
	       sync with local master twice: once to make sure all slaves 
	       have got proctable, and second after the master has synced bm
	    */
	    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());
	    p4_barrier(&(p4_global->cluster_barrier),p4_num_cluster_ids());

	    p4_dprintfl(20, "remote process starting\n");
            ALOG_SETUP(p4_local->my_id,ALOG_TRUNCATE);
            ALOG_LOG(p4_local->my_id,BEGIN_USER,0,"");
	    slave();
            ALOG_LOG(p4_local->my_id,END_USER,0,"");
            ALOG_OUTPUT;
	    p4_dprintfl(20, "remote process exiting\n");

	    exit(0);
	}

	/* Send off the slave info to the bm */
	bm_msg.type = p4_i_to_n(REMOTE_SLAVE_INFO);
	bm_msg.slave_idx = p4_i_to_n(slave_idx);
	bm_msg.slave_pid = p4_i_to_n(slave_pid);
	bm_msg.switch_port = p4_i_to_n(rm_switch_port);
	strcpy(bm_msg.machine_type,P4_MACHINE_TYPE);
	net_send(bm_fd, &bm_msg, sizeof(struct bm_rm_msg), FALSE);

	g->local_slave_count++;
    }
#endif

    /* Send the end message to the bm */
    bm_msg.type = p4_i_to_n(REMOTE_SLAVE_INFO_END);
    net_send(bm_fd, &bm_msg, sizeof(struct bm_rm_msg), FALSE);

    /*
     * Done creating slaves. Now fork off the listener .. we've already
     * created the socket and bound a port to it
     */

#   if !defined(IPSC860)  &&  !defined(CM5)
    g->listener_port = listener_port;
    p4_local->listener_fd = end_1;
    listener_pid = fork_p4();
    if (listener_pid == 0)
    {
	/* Inside listener */
	sprintf(whoami, "rm_l_%d_%d", rm_num, getpid());
	p4_dprintfl(70, "inside listener pid %d\n", getpid());
	p4_local = alloc_local_listener();
	l->listening_fd = listener_fd;
	l->slave_fd = end_2;
	close(end_1);
	listener();
	exit(0);
    }

    /* Else we're still in the remote master */

    p4_dprintfl(70, "created listener pid %d\n", listener_pid);
    /* We need to close the fds from the listener setup */
    close(listener_fd);
    close(end_2);
    g->listener_pid = listener_pid;
#   endif
}


P4VOID receive_proc_table(bm_fd)
int bm_fd;
{
    P4BOOL done;
    struct bm_rm_msg msg;
    int type;
    int port, unix_id, slave_idx, group_id;
    int switch_port;

    p4_dprintfl(90, "receive_proc_table\n");
    for (done = FALSE; !done;)
    {
	if (net_recv(bm_fd, &msg, sizeof(msg)) == PRECV_EOF)
	    p4_error("recv_proc_table: got net_send_eof", bm_fd);

	type = p4_n_to_i(msg.type);
	switch (type)
	{
	  case PROC_TABLE_ENTRY:
	    group_id = p4_n_to_i(msg.group_id);
	    port = p4_n_to_i(msg.port);
	    unix_id = p4_n_to_i(msg.unix_id);
	    slave_idx = p4_n_to_i(msg.slave_idx);
	    switch_port = p4_n_to_i(msg.switch_port);
	    p4_dprintfl(90, "got entry gid=%d host=%s port=%d unix_id=%d slave_idx=%d switch_port=%d\n",
			group_id,msg.host_name,port,unix_id,slave_idx,switch_port);
	    /* remote master loading proctable from big master */
	    install_in_proctable(group_id, port, unix_id, msg.host_name,
				 slave_idx, msg.machine_type, switch_port);
	    break;

	  case PROC_TABLE_END:
	    done = TRUE;
	    break;

	  default:
	    p4_dprintf("receive_proc_table: got invalid message type %d\n", type);
	    break;
	}
    }
}
SHAR_EOF
if test -f 'p4_patchlevel.h'
then
	echo shar: over-writing existing file "'p4_patchlevel.h'"
fi
cat << \SHAR_EOF > 'p4_patchlevel.h'
#define P4_PATCHLEVEL "1.2"
SHAR_EOF
cd ..
if test ! -d 'lib_f'
then
	mkdir 'lib_f'
fi
cd 'lib_f'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains the Fortran interface to p4.  

The principal files are:


p4_fort.c    
p4_fmaster.c - contain of the Fortran-callable subroutines that 
               interface to the p4 C functions.  They define the 
	       "Fortran interface".


p4_fmain.f   - a Fortran top-level for remote slaves, so that they 
               can do Fortran I/O.

p4_fargs.f   - some Fortran utility routines.
SHAR_EOF
if test -f 'interface'
then
	echo shar: over-writing existing file "'interface'"
fi
cat << \SHAR_EOF > 'interface'
*   int p4_initenv();
*   int p4_create_procgroup();
*   int p4_wait_for_end();

*   int p4_get_my_id();
*   int p4_num_total_ids();

*   int p4_get_my_cluster_id();
*   int p4_num_cluster_ids();
*   VOID p4_get_cluster_ids():

*   int p4_broadcast();
*   int p4_broadcastx();
*   int p4_send();
*   int p4_sendx();
*   int p4_sendr();
*   int p4_sendrx();
*   BOOL p4_messages_available();
*   int p4_recv();
    int p4_global_op();

*   int p4_clock();
*   int p4_ustimer();
*   char *p4_version();

*   int p4_soft_errors();
*   VOID p4_error();
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

FORTLIBDIR = $(P4_HOME_DIR)/lib_f

P4_COMM_OBJ = p4_fargs.o p4_fort.o p4_fmaster.o

P4_CUBENODE_OBJ = $(P4_COMM_OBJ)

P4_CUBEHOST_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ)

default:$(P) all

all:$(P) libp4_f.a p4_fmain.o

libp4_f.a:$(P) $(P4_OBJ)
	$(AR) ../lib_f/$@ $?
	$(RANLIB) ../lib_f/$@

realclean: clean
	$(RM) -f Makefile libp4_f.a
SHAR_EOF
if test -f 'p4_fargs.f'
then
	echo shar: over-writing existing file "'p4_fargs.f'"
fi
cat << \SHAR_EOF > 'p4_fargs.f'
      subroutine args(i,argv)
C reads the options from the command line
      integer i
C DO NOT declare argv any bigger than 80
      character*80 argv
      call getarg(i,argv)
      end

      subroutine numargc(icnt)
C retrieves the number of command line arguments
      integer icnt
      icnt = iargc()
      end
SHAR_EOF
if test -f 'p4_fmain.f'
then
	echo shar: over-writing existing file "'p4_fmain.f'"
fi
cat << \SHAR_EOF > 'p4_fmain.f'
C     p4_fmain.f
C This file is the main for all the remote slave processes.  This
C permits fortran I/O from the remotes slaves.
      program fmain
C     print *,'enter fortran slave'
      call slstart()
C     print *,'exit fortran slave'
      end
SHAR_EOF
if test -f 'p4_fmaster.c'
then
	echo shar: over-writing existing file "'p4_fmaster.c'"
fi
cat << \SHAR_EOF > 'p4_fmaster.c'
#include "p4.h"
#include "p4_sys.h"
#include "p4_fort.h"

VOID p4init_()
{
    int argc = 0;
    char *argv[20];
    char *q;
    int argcnt;
    
    p4_dprintfl(30, "enter fortran p4init\n");
    numargc_(&argcnt);
    while(argc <= argcnt)
    {
	q = (char *)malloc(200);
	/* args Fortran subroutine */
	args_(&argc,q);
	argv[argc] = q;
	q = (char *)index(argv[argc],' ');
	*q = '\0';
	p4_dprintfl(30, "\n fortran argv[%d] = %s\n",argc,argv[argc]);
	++argc;
    }
    p4_initenv(&argc,argv);
    p4_dprintfl(30, "exit fortran p4init\n");
}

VOID p4crpg_()
{
    if (p4_create_procgroup() < 0)
	p4_error("p4crpg_: p4_create_procgroup failed",0);
}

VOID p4cleanup_()
{
    p4_wait_for_end();
}
SHAR_EOF
if test -f 'p4_fort.c'
then
	echo shar: over-writing existing file "'p4_fort.c'"
fi
cat << \SHAR_EOF > 'p4_fort.c'
#include "p4.h"
#include "p4_sys.h"
#include "p4_fort.h"

VOID p4sendr_(type,dest,msg,len,rc)
int *type, *dest, *len;
char *msg;
int *rc;
{
    p4_dprintfl(20,"in p4sendr_, type=%d, dest=%d, len=%d\n",
		*type, *dest, *len);
    *rc = p4_sendr(*type, *dest, msg, *len);
}

VOID p4sendrx_(type,dest,msg,len,data_type,rc)
int *type, *dest, *data_type, *len;
char *msg;
int *rc;
{
    p4_dprintfl(20,"in p4sendrx_, type=%d, dest=%d, dtype=%d len=%d\n",
		*type, *dest, *data_type, *len);
    *rc = p4_sendrx(*type, *dest, msg, *len, *data_type);
}

VOID p4send_(type,dest,msg,len,rc)
int *type, *dest, *len;
char *msg;
int *rc;
{
    p4_dprintfl(20,"in p4send_, type=%d, dest=%d, len=%d\n",
		*type, *dest, *len);
    *rc = p4_send(*type, *dest, msg, *len);
}

VOID p4sendx_(type,dest,msg,len,data_type,rc)
int *type, *dest, *data_type, *len;
char *msg;
int *rc;
{
    p4_dprintfl(20,"in p4sendx_, type=%d, dest=%d, dtype=%d len=%d\n",
		*type, *dest, *data_type, *len);
    *rc = p4_sendx(*type, *dest, msg, *len, *data_type);
}

VOID p4recv_(type,from,buf,buflen,msglen,rc)
int *type, *from;
char *buf;
int *buflen, *msglen, *rc;
{
    char *temp_buf;
    int temp_len;

    p4_dprintfl(20, "p4_recv_: receiving, type=%d, from=%d, buflen=%d\n",
		*type, *from, *buflen);
    temp_buf = NULL;
    *rc = p4_recv(type,from,&temp_buf,&temp_len);
    if (*rc < 0)
	p4_dprintf("p4recv_: p4_recv failed\n");
    else
    {
	if (temp_len > *buflen)
	{
	    *msglen = *buflen;
	    *rc = 1;
	}
	else
	    *msglen = temp_len;
	bcopy(temp_buf,buf,*msglen);
	p4_msg_free(temp_buf);
    }
    p4_dprintfl(20, "p4_recv_: received, len=%d\n",*msglen);
}
  
VOID p4brdcst_(type,data,len,rc)
int *type, *len;
char *data;
int *rc;
{
    *rc = p4_broadcast(*type, data, *len);
}

VOID p4brdcstx_(type,data,len,data_type,rc)
int *type, *data_type, *len;
char *data;
int *rc;
{
    *rc = p4_broadcastx(*type, data, *len, *data_type);
}

VOID p4probe_(type,from,rc)
int *type, *from, *rc;
{
    *rc = (int) p4_messages_available(type, from);
}

int p4myclid_()
{
    return p4_get_my_cluster_id();
}

int p4nclids_()
{
    return p4_num_cluster_ids();
}

VOID p4globarr_(type)
int *type;
{
    (VOID) p4_global_barrier(*type);
}

VOID p4getclmasts_(numids,ids)
int *numids,*ids;
{
    (VOID) p4_get_cluster_masters(numids,ids);
}

VOID p4getclids_(start,end)
int *start,*end;
{
    (VOID) p4_get_cluster_ids(start,end);
}

int p4myid_()
{
    return p4_get_my_id();
}

int p4clock_()
{
    return p4_clock();
}

int p4ustimer_()
{
    return p4_ustimer();
}

int p4ntotids_()
{
    return p4_num_total_ids();
}

int p4nslaves_()
{
    return p4_num_total_slaves();
}

VOID p4error_(str,val)
char *str;
int *val;
{
    (VOID) p4_error(str,*val);
}

VOID p4avlbufs_()
{
    p4_print_avail_buffs();
}

VOID p4setavlbuf_(idx,size)
int *idx, *size;
{
    p4_set_avail_buff((*idx)-1,*size);
}

VOID p4softerrs_(new,old)
int new,*old;
{
    *old = (int) p4_soft_errors(new);
}

VOID p4version_()
{
    printf("p4version %s\n",p4_version());
}

VOID p4globop_(type, x, nelem, size, op, data_type, rc)
int *type;
char *x;
int *nelem;
int *size;
int *data_type;
int *rc;
VOID (*op)();
{
int *opind;

    *rc = p4_global_op(*type, x, *nelem, *size, op, *data_type);
}

VOID p4dblsumop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_sum_op(a,b,n);
}

VOID p4dblmultop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_mult_op(a,b,n);
}

VOID p4dblmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_max_op(a,b,n);
}

VOID p4dblminop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_min_op(a,b,n);
}

VOID p4dblabsmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_absmax_op(a,b,n);
}

VOID p4dblabsminop_(a,b,n)
char *a,*b;
int n;
{
    p4_dbl_absmin_op(a,b,n);
}

VOID p4fltsumop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_sum_op(a,b,n);
}

VOID p4fltmultop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_mult_op(a,b,n);
}

VOID p4fltmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_max_op(a,b,n);
}

VOID p4fltminop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_min_op(a,b,n);
}

VOID p4fltabsmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_absmax_op(a,b,n);
}

VOID p4fltabsminop_(a,b,n)
char *a,*b;
int n;
{
    p4_flt_absmin_op(a,b,n);
}

VOID p4intsumop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_sum_op(a,b,n);
}

VOID p4intmultop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_mult_op(a,b,n);
}

VOID p4intmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_max_op(a,b,n);
}

VOID p4intminop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_min_op(a,b,n);
}

VOID p4intabsmaxop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_absmax_op(a,b,n);
}

VOID p4intabsminop_(a,b,n)
char *a,*b;
int n;
{
    p4_int_absmin_op(a,b,n);
}

VOID slave()
{
    fslave_();
}

VOID p4flush_()
{
    printf("\n");
    fflush(stdout);
}

VOID slstart_()
{
    int argc;
    char *argv[10];
    char *q;

    p4_dprintfl(30, "enter fortran slstart \n");
    for(argc=0; argc<4; argc++)
    {
	q = (char *)malloc(200);
	/* args Fortran subroutine */
	args_(&argc,q);
	argv[argc] = q;
	q = (char *)index(argv[argc],' ');
	*q = '\0';
	p4_dprintfl(30,"\n fortran slstart argv[%d] = :%s:\n",
			argc,argv[argc]);
    }
    p4_dprintfl(30, "slstart len of argv[1] = %d\n",strlen(argv[1]));
    /*******
    _F_init(argc,argv,0,0);
    *******/

    p4_initenv(&argc, argv);
    slave();
    p4_wait_for_end();

    p4_dprintfl(30, "exit fortran slstart\n");
}


SHAR_EOF
if test -f 'p4_fort.h'
then
	echo shar: over-writing existing file "'p4_fort.h'"
fi
cat << \SHAR_EOF > 'p4_fort.h'
#if defined(CRAY) || defined(TITAN)

#define p4send_ P4SEND
#define p4sendx_ P4SENDX
#define p4sendr_ P4SENDR
#define p4sendrx_ P4SENDRX
#define p4recv_ P4RECV
#define p4probe_ P4PROBE
#define p4ntotids_ P4NTOTIDS
#define p4nslaves_ P4NSLAVES
#define p4nclids_ P4NCLIDS
#define p4myid_ P4MYID
#define p4myclid_ P4MYCLID
#define p4globarr_ P4GLOBARR
#define p4getclmasts_ P4GETCLMASTS
#define p4getclids_ P4GETCLIDS
#define p4clock_ P4CLOCK
#define p4ustimer_ P4USTIMER
#define p4flush_ P4FLUSH
#define p4init_ P4INIT
#define p4crpg_ P4CRPG
#define p4cleanup_ P4CLEANUP
#define p4brdcst_ P4BRDCST
#define p4brdcstx_ P4BRDCSTX
#define p4error_ P4ERROR
#define p4softerrs_ P4SOFTERRS
#define p4version_ P4VERSION
#define p4avlbufs_ P4AVLBUFS
#define p4setavlbuf_ P4SETAVLBUF
#define p4globop_ P4GLOBOP
#define p4dblsumop_ P4DBLSUMOP
#define p4dblmultop_ P4DBLMULTOP
#define p4dblmaxop_ P4DBLMAXOP
#define p4dblminop_ P4DBLMINOP
#define p4dblabsmaxop_ P4DBLABSMAXOP
#define p4dblabsminop_ P4DBLABSMINOP
#define p4fltsumop_ P4FLTSUMOP
#define p4fltmultop_ P4FLTMULTOP
#define p4fltmaxop_ P4FLTMAXOP
#define p4fltminop_ P4FLTMINOP
#define p4fltabsmaxop_ P4FLTABSMAXOP
#define p4fltabsminop_ P4FLTABSMINOP
#define p4intsumop_ P4INTSUMOP
#define p4intmultop_ P4INTMULTOP
#define p4intmaxop_ P4INTMAXOP
#define p4intminop_ P4INTMINOP
#define p4intabsmaxop_ P4INTABSMAXOP
#define p4intabsminop_ P4INTABSMINOP

#define fslave_ FSLAVE
#define slstart_ SLSTART
#define args_ ARGS
#define numargc_ NUMARGC

#endif

#if defined(NEXT)

#define p4send_ p4send
#define p4sendx_ p4sendx
#define p4sendr_ p4sendr
#define p4sendrx_ p4sendrx
#define p4recv_ p4recv
#define p4probe_ p4probe
#define p4ntotids_ p4ntotids
#define p4nslaves_ p4nslaves
#define p4nclids_ p4nclids
#define p4myid_ p4myid
#define p4myclid_ p4myclid
#define p4globarr_ P4globarr
#define p4getclmasts_ P4getclmasts
#define p4getclids_ p4getclids
#define p4clock_ p4clock
#define p4ustimer_ p4ustimer
#define p4flush_ p4flush
#define p4init_ p4init
#define p4crpg_ p4crpg
#define p4cleanup_ p4cleanup
#define p4brdcst_ p4brdcst
#define p4brdcstx_ p4brdcstx
#define p4error_ p4error
#define p4softerrs_ p4softerrs
#define p4version_ p4version
#define p4avlbufs_ p4avlbufs
#define p4setavlbuf_ p4setavlbuf
#define p4globop_ p4globop
#define p4dblsumop_ p4dblsumop
#define p4dblmultop_ p4dblmultop
#define p4dblmaxop_ p4dblmaxop
#define p4dblminop_ p4dblminop
#define p4dblabsmaxop_ p4dblabsmaxop
#define p4dblabsminop_ p4dblabsminop
#define p4fltsumop_ p4fltsumop
#define p4fltmultop_ p4fltmultop
#define p4fltmaxop_ p4fltmaxop
#define p4fltminop_ p4fltminop
#define p4fltabsmaxop_ p4fltabsmaxop
#define p4fltabsminop_ p4fltabsminop
#define p4intsumop_ p4intsumop
#define p4intmultop_ p4intmultop
#define p4intmaxop_ p4intmaxop
#define p4intminop_ p4intminop
#define p4intabsmaxop_ p4intabsmaxop
#define p4intabsminop_ p4intabsminop

#define fslave_ fslave
#define slstart_ slstart
#define args_ args
#define numargc_ numargc

#endif
SHAR_EOF
if test -f 'p4f.h'
then
	echo shar: over-writing existing file "'p4f.h'"
fi
cat << \SHAR_EOF > 'p4f.h'
      integer p4myid
      integer p4ntotids
      integer p4nslaves
      integer p4myclid
      integer p4nclids
      integer p4clock
      integer p4ustimer

      integer P4NOX, P4INT, P4LNG, P4FLT, P4DBL
      parameter (P4NOX = 0, P4INT = 1, P4LNG = 2)
      parameter (P4FLT = 3, P4DBL = 4)

      external p4dblsumop
      external p4dblmultop
      external p4dblmaxop
      external p4dblminop
      external p4dblabsmaxop
      external p4dblabsminop

      external p4fltsumop
      external p4fltmultop
      external p4fltmaxop
      external p4fltminop
      external p4fltabsmaxop
      external p4fltabsminop

      external p4intsumop
      external p4intmultop
      external p4intmaxop
      external p4intminop
      external p4intabsmaxop
      external p4intabsminop
SHAR_EOF
if test -f 'p4_args.c'
then
	echo shar: over-writing existing file "'p4_args.c'"
fi
cat << \SHAR_EOF > 'p4_args.c'
#include "p4.h"
#include "p4_sys.h"

#if defined(NEXT)  
#define ARGC  _NXArgc
#define ARGV  _NXArgv
#endif

#if defined(IBM3090)
#endif

#if defined(NEXT)  ||  defined(IBM3090)

extern int ARGC;
extern char **ARGV;

int numargc_()
{
    return(ARGC);
}

int args_(i,arg)
int i;
char *arg;
{
    strcpy(arg,ARGV[i]);
}

#endif
SHAR_EOF
cd ..
if test ! -d 'messages'
then
	mkdir 'messages'
fi
cd 'messages'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains some examples of message-passing programs.  Each has a
master part and a slave part.  There are also a number of sample procgroup
files.

  sr	- simple ring communication.  Prompts for message.
  sr1   - prompts for a message size and number of times to circulate the
	  message around the ring.
  grid  - message passing version of Dirichlet problem.
	  currently requires 4 processes in procgroup file.

  *.pg	- The procgroup files necessary for running the above programs on
          various configurations of machines.



The bigrun stuff is what I used to run 5 processes, 
    1 on a sun (delilah), 2 on a gamma (ajax), and 2 on delta1
Note that to compile the gamma and delta code, you must reset 
PGI before each make.

Also, BIG NOTE:  in the lib subdirectories, you must change the 
machine type to IPSC860 before making.
SHAR_EOF
if test -f 'grid_comm.c'
then
	echo shar: over-writing existing file "'grid_comm.c'"
fi
cat << \SHAR_EOF > 'grid_comm.c'
#define BOOL            int
#define TRUE            1
#define FALSE           0

#define ROWS		 100
#define COLUMNS		 100
#define ROWS_PER_SUB     50
#define COLUMNS_PER_SUB  50

#define PROCS_PER_COL	(ROWS / ROWS_PER_SUB)
#define PROCS_PER_ROW	(COLUMNS / COLUMNS_PER_SUB)
#define N_PROCS		(PROCS_PER_ROW * PROCS_PER_COL)

upper_bound(row)
int row;
{
    return((row == 1));
}

lower_bound(row)
int row;
{
    return((row == (ROWS - ROWS_PER_SUB + 1)));
}

right_bound(column)
int column;
{
    return((column == (COLUMNS - COLUMNS_PER_SUB + 1)));
}

left_bound(column)
int column;
{
    return((column == 1));
}
SHAR_EOF
if test -f 'grid_gamma.pg'
then
	echo shar: over-writing existing file "'grid_gamma.pg'"
fi
cat << \SHAR_EOF > 'grid_gamma.pg'
backend 0
+  4  /Net/anagram/anagram1/lusk/gamma/p4.node/messages/grid_slave
SHAR_EOF
if test -f 'grid_master.c'
then
	echo shar: over-writing existing file "'grid_master.c'"
fi
cat << \SHAR_EOF > 'grid_master.c'
#include "p4.h"

#ifndef TRUE
#define BOOL            int
#define TRUE            1
#define FALSE           0
#endif

/*  the following numbers define a grid of 4 processes, plus one master */
#define ROWS		100
#define COLUMNS		100

#define ROWS_PER_SUB 	50
#define COLUMNS_PER_SUB 50



#define PROCS_PER_COL	(ROWS / ROWS_PER_SUB)
#define PROCS_PER_ROW	(COLUMNS / COLUMNS_PER_SUB)
#define N_PROCS		(PROCS_PER_ROW * PROCS_PER_COL)

/* message types */
#define CNTL            0
#define C_BOUNDARY	1
#define R_BOUNDARY	2
#define ANSWER      	3

#define MASTER		0    /*master proc id */

struct cntl_rec {
  	int row;
	int col;
        int upper_neighbor;
	int right_neighbor;
	int lower_neighbor;
	int left_neighbor;
	int iterations;
	double bounded_subgrid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
	};

struct c_boundary {
	double col[ROWS_PER_SUB];
	};		

struct r_boundary {
       	double row[COLUMNS_PER_SUB];
	};

struct answer_rec {
	double subgrid[ROWS_PER_SUB][COLUMNS_PER_SUB];
        };

double avggrid();
double avgbnd();

/* some compilers require this to be outside main (too many locals) */
double  grid[ROWS+2][COLUMNS+2];


main(argc,argv)
int argc;
char *argv[];
{

    struct cntl_rec rec1;
    struct answer_rec *rec2;

    int		process_id[N_PROCS], proc_id;
    int 	proc, i, j, f_row, f_col, ln, msg_type;
    int timestart, timeend;
    double avg;

    p4_initenv(&argc,argv);
    p4_create_procgroup();

    /*  old way - RL
    gridinit(grid,ROWS+2,COLUMNS+2);
    */
    gridinit(grid,ROWS,COLUMNS);

    printf("Enter the number of iterations: ");
    scanf("%d",&(rec1.iterations));		

    timestart = p4_clock();

    for (proc=0; proc < N_PROCS; proc++)
	process_id[proc] = proc+1;

    for (proc=0; proc < N_PROCS; proc++)
    {
	rec1.row = first_row(proc);
	rec1.col = first_column(proc);
	rec1.left_neighbor = 0;
	rec1.right_neighbor = 0;
	rec1.upper_neighbor = 0;
	rec1.lower_neighbor = 0;
	if (!upper_bound(rec1.row)) {
	    i = rec1.row - ROWS_PER_SUB;
	    j = which_proc(i,rec1.col);
	    rec1.upper_neighbor = process_id[j];
	}
	if (!lower_bound(rec1.row)) {
	    i = rec1.row + ROWS_PER_SUB;
	    j = which_proc(i,rec1.col);
	    rec1.lower_neighbor = process_id[j];
	}
	if (!right_bound(rec1.col)) {
	    i = rec1.col + COLUMNS_PER_SUB;
	    j = which_proc(rec1.row,i);
	    rec1.right_neighbor = process_id[j];
	}
	if (!left_bound(rec1.col)) {
	    i = rec1.col - COLUMNS_PER_SUB;
	    j = which_proc(rec1.row,i);
	    rec1.left_neighbor = process_id[j];
	}

	for (i=(rec1.row - 1); i <= (rec1.row + ROWS_PER_SUB); i++) 
	{

	    for (j=(rec1.col - 1); j <= (rec1.col + COLUMNS_PER_SUB); j++)
	    {
		rec1.bounded_subgrid[(i-(rec1.row-1))][(j-(rec1.col-1))] =
		    grid[i][j];
	    }
	}
	ln = sizeof(struct cntl_rec);
	p4_sendr(CNTL,process_id[proc],&rec1,ln);
    }

    for (proc=0; proc < N_PROCS; proc++)
    {
	p4_dprintfl(5,"master receiving answer\n");
	
	msg_type = ANSWER;
	proc_id = -1; 
	rec2 = NULL;
	p4_recv(&msg_type,&proc_id,&rec2,&ln);
	p4_dprintfl(5,"master received answer from slave %d\n",proc_id);
	f_row = first_row(proc_id - 1);
	f_col = first_column(proc_id - 1);
	for (i=0; i < ROWS_PER_SUB; i++) 
	    for (j=0; j < COLUMNS_PER_SUB; j++)
		grid[f_row + i][f_col + j] = rec2->subgrid[i][j];
    }
    timeend = p4_clock();
    p4_wait_for_end();
    printf("total time %.3f seconds\n",(timeend - timestart)/1000.0);

    /* printgrid(grid,ROWS,COLUMNS); */
    avg = avggrid(grid,ROWS,COLUMNS);
    printf("average value of grid = %f\n",avg);
}

first_row(proc)
int proc;
{
    return(((proc / PROCS_PER_ROW) * ROWS_PER_SUB) + 1);
}

first_column(proc)
int proc;
{
    return(((proc % PROCS_PER_ROW) * COLUMNS_PER_SUB) + 1);
}

which_proc(row,column)
int row,column;
{
    return((((row - 1) / ROWS_PER_SUB) * PROCS_PER_ROW) + 
	   ((column - 1) / COLUMNS_PER_SUB));
}

/* old way - RL
gridinit(m,x,y)
double m[ROWS+2][COLUMNS+2];
int x,y;
{
    int i, j;

    for (i=0; i <= x-1; i++)
        for (j=0; j <= y-1; j++)
            m[i][j] = 0;

    for (j=0; j < y; j++)
    {
        m[0][j] = phi(1,j+1);
        m[x-1][j]= phi(x+1,j+1);
    }
    for (i=0; i < x; i++)
    {
        m[i][0] = phi(i+1,1);
        m[i][y-1] = phi(i+1,y+1);
    }
}
*/


gridinit(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r, c;
{
    int i, j;
    double bndavg;
    
    for (j=0; j < (c + 2); j++)
    {
        m[0][j] = phi(1,j+1);
        m[r+1][j]= phi(r+2,j+1);
    }
    for (i=1; i < (r + 2); i++)
    {
        m[i][0] = phi(i+1,1);
        m[i][c+1] = phi(i+1,c+2);
    }
    bndavg = avgbnd(m,r,c);
    printf("boundary average = %f\n",bndavg);

    /* initialize the interior of the grids to the average over the boundary*/
    for (i=1; i <= r; i++)
        for (j=1; j <= c; j++)
            /* m[i][j] = bndavg; this optimization hinders debugging */
	    m[i][j] = 0;
}

double avggrid(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    avg += m[i][j];
    return(avg/((r+2)*(c+2)));
}

double avgbnd(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	    avg += m[i][0];
    for (i = 0; i < (r+2); i++)
	    avg += m[i][c+1];
    for (i = 1; i < (c+1); i++)
	    avg += m[0][i];
    for (i = 1; i < (c+1); i++)
	    avg += m[r+1][i];
    return(avg/(2*(c+2) + 2*(r+2) - 4)); /* average over boundary */
}

phi(x,y)
int x,y;
{
	return(x*x-y*y+x*y);
}

printgrid(grid,r,c)
double grid[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;

    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    printf("grid[%3d][%3d] = %10.5f\n",i,j,grid[i][j]);
}

SHAR_EOF
if test -f 'grid_slave.c'
then
	echo shar: over-writing existing file "'grid_slave.c'"
fi
cat << \SHAR_EOF > 'grid_slave.c'
#include "p4.h"
    
#ifndef TRUE
#define BOOL            int
#define TRUE            1
#define FALSE           0
#endif
    
#define ROWS	         100
#define COLUMNS	         100
#define ROWS_PER_SUB     50
#define COLUMNS_PER_SUB	 50

#define MASTER		  0
    
#define PROCS_PER_COL	(ROWS / ROWS_PER_SUB)
#define PROCS_PER_ROW	(COLUMNS / COLUMNS_PER_SUB)
#define N_PROCS		(PROCS_PER_ROW * PROCS_PER_COL)

/* message types */
#define CNTL            0
#define C_BOUNDARY	1
#define R_BOUNDARY	2
#define ANSWER      	3

/* log event types */
#define SEND            102
#define RECEIVE         101
#define CREATE          100
#define WAIT_MSG        6
#define WORKING         4
#define DONE_WORK       5

struct cntl_rec {
    int row;
    int col;
    int upper_neighbor;
    int right_neighbor;
    int lower_neighbor;
    int left_neighbor;
    int iterations;
    double bounded_subgrid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
};

struct c_boundary {
    double col[ROWS_PER_SUB];
};

struct r_boundary {
    double row[COLUMNS_PER_SUB];
};

struct answer_rec {
    double subgrid[ROWS_PER_SUB][COLUMNS_PER_SUB];
};


/* some compilers require this to be outside slave (too many locals) */
double grid1[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
double grid2[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];

slave()
{
    struct cntl_rec *rec1;
    double *current_grid, *next_grid, *temp;
    int master_id, proc_id, msg_type;
    int i, j, n, ln;
    
    master_id = MASTER;
    msg_type = CNTL;
    rec1 = NULL;
    p4_recv(&msg_type,&master_id, &rec1, &ln);
    for (i=0; i < (ROWS_PER_SUB + 2); i++) 
        for (j=0; j < (COLUMNS_PER_SUB + 2); j++)
	{
	    grid2[i][j] = rec1->bounded_subgrid[i][j];
	    grid1[i][j] = rec1->bounded_subgrid[i][j];
	};
    
    current_grid = (double *) grid1;
    next_grid = (double *) grid2;
    p4_dprintfl(5,"in slave, iterations = %d\n",rec1->iterations);
    for (n=rec1->iterations, i=0; i < n; i++)
    {
	/*
	if ((rec1->row == 1) && (rec1->col == 1))
	    printf("current subgrid at %d,%d, after %d iterations\n",
		   rec1->row,rec1->col,i);
        printsubgrid(current_grid,ROWS_PER_SUB,COLUMNS_PER_SUB);
	*/
	compute1_iter(current_grid,next_grid);
	if (!((i + 1) == n))   /* if not the last iteration */
	{
	    if (!upper_bound(rec1->row))
		send_row(next_grid, 1, rec1->upper_neighbor);
	    if (!lower_bound(rec1->row))
		send_row(next_grid, ROWS_PER_SUB, rec1->lower_neighbor);
	    if (!left_bound(rec1->col))
		send_col(next_grid, 1, rec1->left_neighbor);
	    if (!right_bound(rec1->col))
		send_col(next_grid, COLUMNS_PER_SUB, rec1->right_neighbor);
	    if (!lower_bound(rec1->row))
		receive_row(next_grid, (ROWS_PER_SUB+1),rec1->lower_neighbor);
	    if (!upper_bound(rec1->row))
		receive_row(next_grid, 0, rec1->upper_neighbor);
	    if (!right_bound(rec1->col))
		receive_col(next_grid,(COLUMNS_PER_SUB+1),rec1->right_neighbor);
	    if (!left_bound(rec1->col))
		receive_col(next_grid, 0, rec1->left_neighbor);
	}
	/* swap grids */
	temp = current_grid;
	current_grid = next_grid;
	next_grid = temp;
    }
    send_answer(current_grid, master_id);
}  

send_row(grid, row, proc_id)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int row, proc_id;
{
    int x, ln, msg_type;
    struct r_boundary m1;
    
    msg_type = R_BOUNDARY;
    for (x=1; x <= COLUMNS_PER_SUB; x++)
	m1.row[x-1] = grid[row][x];
    ln = sizeof(struct r_boundary);
    p4_send(msg_type, proc_id, &m1, ln);
}

send_col(grid, col, proc_id)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int col;
int proc_id;
{
    struct c_boundary m1;
    int i, ln, msg_type;
    
    msg_type = C_BOUNDARY;
    ln = sizeof(struct c_boundary);
    for (i=1; i <= ROWS_PER_SUB; i++)
	m1.col[i-1] = grid[i][col];
    p4_send(msg_type, proc_id, &m1, ln);
}

receive_row(grid, row, proc_id)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int row;
int proc_id;
{
    struct r_boundary *m1;
    int sender, mln, x, msg_type;
    
    msg_type = R_BOUNDARY;
    sender   = proc_id;
    m1 = NULL;
    p4_recv(&msg_type,&sender, &m1, &mln);
    for (x=1; x <= COLUMNS_PER_SUB; x++)
	grid[row][x] = m1->row[x-1];
    p4_msg_free(m1);
}

receive_col(grid, col, proc_id)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int col;
int proc_id;
{
    struct c_boundary *m1;
    int sender, mln, x, msg_type;
    
    msg_type = C_BOUNDARY;
    sender = proc_id;
    m1 = NULL;
    p4_recv(&msg_type,&sender, &m1, &mln);
    for (x=1;x <= ROWS_PER_SUB; x++)
	grid[x][col] = m1->col[x-1];
    p4_msg_free(m1);
}


send_answer(grid, master_id)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int master_id;
{
    struct answer_rec rec1;
    int i, j, ln, msg_type;
    
    msg_type = ANSWER;
    for (i=1; i <= ROWS_PER_SUB; i++)
	for (j=1; j <= COLUMNS_PER_SUB; j++)
	    rec1.subgrid[i-1][j-1] = grid[i][j];
    ln = sizeof(struct answer_rec);
    p4_dprintfl(5,"sending answer\n");

    p4_sendr(msg_type,master_id, &rec1, ln);

    p4_dprintfl(5,"sent answer\n");
}

compute1_iter(current,next)
double current[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
double next[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
{
    int i, j;
    
    for (i=1; i <= (ROWS_PER_SUB); i++)
	for (j=1; j <= COLUMNS_PER_SUB; j++)
	    next[i][j] = (current[i-1][j] + 
			  current[i+1][j] + 
			  current[i][j-1] + 
			  current[i][j+1]) / 4.0;
}

printsubgrid(grid,r,c)
double grid[ROWS_PER_SUB+2][COLUMNS_PER_SUB+2];
int r,c;
{
    int i,j;

    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    printf("grid[%3d][%3d] = %10.5f\n",i,j,grid[i][j]);
}
SHAR_EOF
if test -f 'local4.pg'
then
	echo shar: over-writing existing file "'local4.pg'"
fi
cat << \SHAR_EOF > 'local4.pg'
local 4
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIBS = $(LIBDIR)/libp4.a $(MDEP_LIBS)

default:$(P)
	make $(MFLAGS) sr_test sr1_test systest

all:$(P)
	make $(MFLAGS) sr_test systest 
	make $(MFLAGS) sr_master sr_slave 
	make $(MFLAGS) grid_master grid_slave

sr_test:$(P) sr_test.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o sr_test sr_test.o $(LIBS)

sr1_test:$(P) sr1_test.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o sr1_test sr1_test.o $(LIBS)

systest:$(P) systest.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o $@ systest.o $(LIBS)

grid_master:$(P) grid_master.o grid_slave.o grid_comm.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o grid_master grid_master.o grid_slave.o \
	grid_comm.o $(LIBS)

grid_slave:$(P) grid_slave.o grid_comm.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o grid_slave $(LIBDIR)/p4_cmain.o \
	grid_slave.o grid_comm.o $(LIBS)

sr_master:$(P) sr_master.o sr_slave.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o sr_master sr_master.o sr_slave.o $(LIBS)

sr_slave:$(P) sr_slave.o $(LIBDIR)/libp4.a
	$(CLINKER) $(CFLAGS) -o sr_slave $(LIBDIR)/p4_cmain.o sr_slave.o $(LIBS)

realclean: clean
	$(RM) -f sr_test sr1_test systest grid_master grid_slave \
	sr_master sr_slave Makefile 
SHAR_EOF
if test -f 'runcube'
then
	echo shar: over-writing existing file "'runcube'"
fi
cat << \SHAR_EOF > 'runcube'
# script to run sr_test in SPMD mode
#
set echo
#
getcube -t4
#
load 1 2 sr_test -amp4slave
load 0 sr_test -dbg 33 -rdbg 33 ; waitcube << EOFa
foobar
EOFa
#
killcube
relcube
SHAR_EOF
chmod +x 'runcube'
if test -f 'rundelta'
then
	echo shar: over-writing existing file "'rundelta'"
fi
cat << \SHAR_EOF > 'rundelta'
mexec -t "(2,2)" -f "0 sr_test" -f "1 2 sr_test -amp4slave"
SHAR_EOF
chmod +x 'rundelta'
if test -f 'procgroup'
then
	echo shar: over-writing existing file "'procgroup'"
fi
cat << \SHAR_EOF > 'procgroup'
local 1
#anagram 1 /Net/anagram/anagram3/rbutler/p4-1.2/messages/dummy
#skiff 2 /Net/anagram/anagram3/rbutler/p4-1.2/messages/systest
#juju 1 /home/lusk/p4-1.2/messages/systest
#anagram 2 /home/lusk/anagram/p4-1.2/messages/systest
#local 8 /scr3/lusk/p4-1.2/messages/sr_test
#ksr1 1 /home/ksr1/u0/lusk/p4/messages/sr_test
#ksr1 1 /home/ksr1/u0/lusk/p4/messages/sr_test
#ajax 2 /home/delilah6/lusk/p4t1.node/messages/bigrun
#delta1 2 /home/delilah6/lusk/p4t1.delta/messages/bigrun
#anagram 1 /Net/anagram/anagram3/rbutler/p4t1/messages/sr_test
#hp720 1 /users/chelle/rbutler/p4t03/messages/sr_test
#anagram 1 /Net/anagram/anagram8/lusk/p4/messages/sr_slave
#donner 1 /Net/sparky/sparky1/lusk/p4.x/messages/temps
#hurley.tcg.anl.gov 1 /usr8/lusk/p4/messages_newf/slave
# xmp.ctd.anl.gov 1 /n1/lusk/p4/messages/sr_slave
# xmp.ctd.anl.gov 1 /n1/lusk/p4/messages/sr_slave
# for delta or ipsc860:
# local 2 /usr2/lusk/sr_test
SHAR_EOF
if test -f 'sr1_anagram.pg'
then
	echo shar: over-writing existing file "'sr1_anagram.pg'"
fi
cat << \SHAR_EOF > 'sr1_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages/sr1_test 
SHAR_EOF
if test -f 'sr1_gamma.pg'
then
	echo shar: over-writing existing file "'sr1_gamma.pg'"
fi
cat << \SHAR_EOF > 'sr1_gamma.pg'
backend 0
+  1  /Net/anagram/anagram1/lusk/gamma/p4.node/messages/sr1_test
+  1  /Net/anagram/anagram1/lusk/gamma/p4.node/messages/sr1_test
SHAR_EOF
if test -f 'sr_slave.c'
then
	echo shar: over-writing existing file "'sr_slave.c'"
fi
cat << \SHAR_EOF > 'sr_slave.c'
#include "p4.h"
#include "sr_user.h"

slave()	
{
    char buf[100];
    int size;
    int n;
    int nslaves_t;
    int nslaves_l;
    int start, end;
    int type;
    int done;
    int from;
    int next;
    int my_id;
    int my_cl_id;
    int rm_ind;
    char *incoming;
    int start_time,end_time;
    FILE *tempf;
    
    my_id = p4_get_my_id();
    p4_dprintfl(10,"sr_slave:in slave %d\n",my_id);
    /****
    dump_global(5);
    dump_local(5);
    ****/
    nslaves_t = p4_num_total_ids() - 1;

    /***/
    rm_ind = p4_am_i_cluster_master();
    nslaves_l = p4_num_cluster_ids() - 1;
    p4_get_cluster_ids(&start, &end); 
    my_cl_id = p4_get_my_cluster_id();

    p4_dprintfl(5,"p4_num_total_slaves=%d num_cluster_slaves=%d\n",
	    nslaves_t,nslaves_l);
    p4_dprintfl(5,"first_local_id=%d last_local_id=%d\n",start,end);
    p4_dprintfl(5,"my_cluster_id=%d my_id=%d rm=%d\n",
	     my_cl_id,my_id,rm_ind);
    /***/

    p4_dprintfl(5,"sr_s: my group id is %d\n",p4_global->proctable[my_id].group_id);
    if (my_id == nslaves_t)
        next = 0;
    else
	next = my_id + 1;
    done = FALSE;
    while (!done)
    {
        p4_dprintfl(99,"sr_slave recving \n");
	type = -1;
	from = -1;
        start_time = p4_clock();
	incoming = NULL;
	p4_recv(&type,&from, &incoming, &size);
	p4_dprintfl(99,"sr_slave received, from=%d, type = %d\n",from,type);
	if (type == END)
	    done = TRUE;
	else 
	    p4_dprintfl(99,"sr_slave: got buf=\"%s\"\n",incoming);
	p4_dprintfl(99,"sr_slave: slave %d sending to %d\n", my_id,next);
	p4_send(type, next, incoming, size);
        end_time = p4_clock();
	p4_dprintfl(99,"sr_slave: slave %d sent to %d\n",my_id,next);
        p4_dprintfl(5,"total time=%d \n",end_time-start_time);
	p4_msg_free(incoming);
    }
    p4_dprintfl(10,"sr_slave %d exiting\n", p4_get_my_id());
}

SHAR_EOF
if test -f 'sr1_test.c'
then
	echo shar: over-writing existing file "'sr1_test.c'"
fi
cat << \SHAR_EOF > 'sr1_test.c'
#include "p4.h"
#include "sr_user.h"

#define MAX_MESSAGE_SIZE 1000000
char msg[MAX_MESSAGE_SIZE];

int main(argc,argv)
int argc;
char **argv;
{

    p4_initenv(&argc,argv);
    if (p4_get_my_id() == 0)
    {
        p4_create_procgroup();
	master();
    }
    else
    {
        slave();
    }
  
    p4_wait_for_end();
}

    
master()
{
    int nslaves;
    int type, size, id, from;
    int my_id;
    char *incoming;
    int done;
    int msgsize, count;
    int starttime, endtime;
    p4_usc_time_t start_ustime, end_ustime;

    nslaves = p4_num_total_slaves();
    printf("number of slaves = %d\n",nslaves);
    my_id = p4_get_my_id();
    
    done = FALSE;
    while (!done)
    {
	printf("message size: ");
	scanf("%d",&msgsize);
	if (msgsize > MAX_MESSAGE_SIZE)
	{
	    printf("too big;  using %d\n",MAX_MESSAGE_SIZE);
	    msgsize = MAX_MESSAGE_SIZE;
	}
	printf("times around loop (or 0 for end): ");
	scanf("%d",&count);
	
	if (count == 0)
	    done = TRUE;
	else
	{
	    starttime = p4_clock();
	    start_ustime = p4_ustimer();
	    while (count > 0)
	    {
		p4_sendr(DATA, 1, msg, msgsize);
		type = -1;
		from = -1;
		incoming = NULL;
		p4_recv(&type, &from, &incoming, &size);
		p4_msg_free(incoming);
		count--;
	    }
	    end_ustime = p4_ustimer();
	    endtime = p4_clock();
	    printf("time %d milliseconds\n",endtime-starttime);
	    printf("time %d microseconds\n",end_ustime-start_ustime);
	}
    }

    p4_sendr(END, 1, msg, 0);
    type = -1;
    from = -1;
    incoming = NULL;
    p4_recv(&type, &from, &incoming, &size);
    p4_msg_free(incoming);
    p4_wait_for_end();
    printf("master exiting normally\n");
}

slave()	
{
    int nslaves;
    int done;
    int type, from, size;
    int next;
    int my_id;
    char *incoming;
    
    my_id = p4_get_my_id();
    nslaves = p4_num_total_slaves();
    
    if (my_id == nslaves)
        next = 0;
    else
	next = my_id + 1;
    
    done = FALSE;
    while (!done)
    {
	type = -1;
	from = -1;
	incoming = NULL;
	p4_recv(&type,&from, &incoming, &size);
	if (type == END)
	    done = TRUE;
	p4_sendr(type, next, incoming, size);
	p4_msg_free(incoming);
    }
}

SHAR_EOF
if test -f 'sr_anagram.pg'
then
	echo shar: over-writing existing file "'sr_anagram.pg'"
fi
cat << \SHAR_EOF > 'sr_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages/sr_slave 
SHAR_EOF
if test -f 'sr_bfly.pg'
then
	echo shar: over-writing existing file "'sr_bfly.pg'"
fi
cat << \SHAR_EOF > 'sr_bfly.pg'
local 0
lepido.mcs.anl.gov 1 /Net/antares/antares3/lusk/butterfly/p4/messages/sr_slave
#lepido.mcs.anl.gov 1 /Net/antares/antares3/lusk/butterfly/p4/messages/sr_slave


SHAR_EOF
if test -f 'sr_cray.pg'
then
	echo shar: over-writing existing file "'sr_cray.pg'"
fi
cat << \SHAR_EOF > 'sr_cray.pg'
local 0
xmp 1 /n1/lusk/p4/messages/sr_slave
SHAR_EOF
if test -f 'sr_gamma.pg'
then
	echo shar: over-writing existing file "'sr_gamma.pg'"
fi
cat << \SHAR_EOF > 'sr_gamma.pg'
backend 0
+  2  /Net/anagram/anagram1/lusk/gamma/p4.node/messages/sr_slave
+  2  /Net/anagram/anagram1/lusk/gamma/p4.node/messages/sr_slave
SHAR_EOF
if test -f 'sr_master.c'
then
	echo shar: over-writing existing file "'sr_master.c'"
fi
cat << \SHAR_EOF > 'sr_master.c'
#include "p4.h"
#include "sr_user.h"
    
main(argc,argv)
int argc;
char **argv;
{
    char buf[100];
    int i, n;
    int size;
    int slv = 3;
    int next;
    int nslaves_t;
    int nslaves_l;
    int start, end;
    char msg[200];
    int type;
    int id;
    int from;
    int my_id;
    int my_cl_id;
    char *incoming, *cp;
    int start_time,end_time;
    
    p4_initenv(&argc,argv);
    p4_create_procgroup();
    p4_dprintfl(9,"Starting master code.\n");
    
    nslaves_t = p4_num_total_ids() - 1;
    nslaves_l = p4_num_cluster_ids() - 1;
    p4_get_cluster_ids(&start,&end);

    my_id = p4_get_my_id();
    my_cl_id = p4_get_my_cluster_id();
    
    p4_dprintfl(5,"p4_num_total_slaves=%d num_cluster_slaves=%d\n", 
	    nslaves_t,nslaves_l);
    p4_dprintfl(5,"first_local_id=%d last_local_id=%d\n",start,end);
    p4_dprintfl(5,"my_id=%d my_cluster_id=%d\n\n",my_id,my_cl_id);
    p4_dprintfl(5,"sr_m: my group id is %d\n",p4_global->proctable[my_id].group_id);
    
    printf("enter a string:\n");
    while (fgets(msg, sizeof(msg), stdin) != NULL)
    {
        for(cp=msg, i=1; *cp; i++, cp++)
	    if(*cp == '\n')
	    {
		*cp = 0;
		break;
	    }
	p4_dprintfl(99,"sr_master sending %s size=%d\n", msg,i);
	start_time = p4_clock();
	p4_sendr(100, 1, msg, i);
	p4_dprintfl(99,"sr_master receiving...\n");
	type = -1;
	from = -1;
	/**********
	while (!p4_messages_available(&type,&from))
	    ;
	printf("master has msg available: type=%d from=%d\n",type,from);
	**********/
	incoming = NULL;
	p4_recv(&type,&from, &incoming, &size);
	end_time = p4_clock();
	printf("total time=%d \n",end_time-start_time);
	printf("master received :%s: from %d\n", incoming, from);
	p4_dprintfl(10,"master: received from=%d type=%d size=%d buf=%s\n", 
	       from, type, size, incoming);
	p4_msg_free(incoming);
	printf("enter a string:\n");
    }
    

    p4_dprintfl(8,"waiting for end msg\n");

    p4_sendr(END, 1, msg, 0);
    type = -1;
    from = -1;
    incoming = NULL;
    p4_recv(&type, &from, &incoming, &size);
    p4_dprintfl(8,"done  from=%d type=%d size=%d \n", 
	     from, type, size);
    p4_msg_free(incoming);
    
    p4_dprintfl(8,"master entering waitforend\n");
    p4_wait_for_end();
    p4_dprintfl(8,"master past waitforend\n");
    printf("master exiting normally\n");
}
SHAR_EOF
if test -f 'sr_test.c'
then
	echo shar: over-writing existing file "'sr_test.c'"
fi
cat << \SHAR_EOF > 'sr_test.c'
#include "p4.h"
#include "sr_user.h"
    
main(argc,argv)
int argc;
char **argv;
{
    
    p4_initenv(&argc,argv);
    if (p4_get_my_id() == 0)
    {
        p4_create_procgroup();
	master();
    }
    else
    {
	slave();
    }
    p4_wait_for_end();
}

master()
{
    char msg[200];
    int i, my_id, size, type, from;
    char *incoming, *cp;
    int start_time,end_time;
    
    my_id = p4_get_my_id();
    
    printf("enter a string:\n");
    while (fgets(msg, sizeof(msg), stdin) != NULL)
    {
        for(cp=msg, i=1; *cp; i++, cp++)
	    if(*cp == '\n')
	    {
		*cp = 0;
		break;
	    }
	start_time = p4_clock();
	p4_sendr(100, 1, msg, i);
	type = -1;
	from = -1;
	incoming = NULL;
	p4_recv(&type, &from, &incoming, &size);
	end_time = p4_clock();
	printf("total time=%d \n",end_time-start_time);
	printf("master received :%s: from %d\n", incoming, from);
	p4_msg_free(incoming);
	printf("enter a string:\n");
    }
    
    p4_sendr(END, 1, msg, 0);
    type = -1;
    from = -1;
    incoming = NULL;
    p4_recv(&type, &from, &incoming, &size);
    p4_msg_free(incoming);
    
    printf("master exiting normally\n");
}

slave()	
{
    int done;
    int my_id, next, size;
    int num_total_slaves, type, from;
    char *incoming;
    
    my_id = p4_get_my_id();
    num_total_slaves = p4_num_total_slaves();
    if (my_id == num_total_slaves)
        next = 0;
    else
	next = my_id + 1;
    done = FALSE;
    while (!done)
    {
	type = -1;
	from = -1;
	incoming = NULL;
	p4_recv(&type,&from, &incoming, &size);
	if (type == END)
	    done = TRUE;
	p4_send(type, next, incoming, size);
	p4_msg_free(incoming);
    }
}

SHAR_EOF
if test -f 'sr_user.h'
then
	echo shar: over-writing existing file "'sr_user.h'"
fi
cat << \SHAR_EOF > 'sr_user.h'
#define DATA 1
#define END 2
SHAR_EOF
if test -f 'systest.c'
then
	echo shar: over-writing existing file "'systest.c'"
fi
cat << \SHAR_EOF > 'systest.c'
#include "p4.h"

extern void slave();

int main(argc,argv)
     int argc;
     char **argv;
/*
  Generic SPMD master program .. just creates environment
  and then calls slave, just as the slaves do.
  This probably does NOT do what U want on the iPSC.
*/
{
  p4_initenv(&argc,argv);
  if (p4_get_my_id() == 0)
      p4_create_procgroup();

  slave();

  p4_wait_for_end();
}

#define MAX(a,b) (((a)>(b)) ? (a) : (b))

static void Hello();
static void Ring();
static void Stress();
static void Globals();
static int  GlobalReadInteger();

extern void synchronize();

void slave()
/*
  Test and time aspects of the system.
*/
{
  int me = p4_get_my_id();
  int option;

  (void) p4_soft_errors(FALSE);  /* This should be the default */

  p4_dprintfl(00,"%d is alive on a %s\n",me,p4_machine_type());
  fflush(stdout);

  while (1) {

    synchronize(1000);
    /* sleep(1); */          /* To allow output to catch up */

  again:
    if (me == 0) {
      /* Read user input for action */
      (void) printf("\nOptions: 0=quit, 1=Hello, 2=Ring, 3=Stress, ");
      (void) printf("4=Globals : ");
      (void) fflush(stdout);
    }
    option = GlobalReadInteger();
    if ( (option < 0) || (option > 4) )
      goto again;

    switch (option) {
    case 0:
      return;

    case 1:
      Hello();
      break;

    case 2:
      Ring();
      break;

    case 3:
      Stress();
      break;

    case 4:
      Globals();
      break;

    default:
      p4_error("systest: invalid option", option);
      break;
    }
  }
}

static void Hello()
/*
  Everyone exchanges a hello message with everyone else.
  The hello message just comprises the sending and target nodes.
*/
{
  int nproc = p4_num_total_ids();
  int me = p4_get_my_id();
  int type = 1;
  int buffer[2], node, *input, length;

  if (me == 0) {
    (void) printf("\nHello test ... show network integrity\n----------\n\n");
    (void) fflush(stdout);
  }

  for (node = 0; node<nproc; node++) {
    if (node != me) {
      buffer[0] = me;
      buffer[1] = node;
      (void) p4_sendx(type,node,(char *) buffer,sizeof buffer,P4INT);
      input = (int *) NULL;
      (void) p4_recv(&type, &node, (char **) &input, &length);

      if ( (input[0] != node) || (input[1] != me) ) {
	(void) fprintf(stderr, "Hello: %d!=%d or %d!=%d\n",
		       input[0], node, input[1], me);
	p4_error("Mismatch on hello process ids", node);
      }

      (void) p4_dprintfl(00,"Hello from %d to %d\n", me, node);
      (void) fflush(stdout);
    }
  }
}

static void Ring()
     /* Time passing a message round a ring */
{
  int me = p4_get_my_id();
  int nproc = p4_num_total_ids();
  int type = 4;
  int left = (me + nproc - 1) % nproc;
  int right = (me + 1) % nproc;
  char *buffer;
  int start, lenbuf, used, max_len, *msg, msg_len;
  double rate;
  p4_usc_time_t start_ustime, end_ustime, used_ustime;

  /* Find out how big a message to use */

  if (me == 0) {
    (void) printf("\nRing test ... time network performance\n---------\n\n");
    (void) printf("Input maximum message size: ");
    (void) fflush(stdout);
  }
  max_len = GlobalReadInteger();
  if ( (max_len <= 0) || (max_len >= 4*1024*1024) )
    max_len = 512*1024;
  if (me == 0)
    if ( (buffer = p4_shmalloc((unsigned) max_len)) == (char *) NULL)
      p4_error("Ring: failed to allocate buffer",max_len);

  type = 5;

  lenbuf = 1;
  while (lenbuf <= max_len) {

    start = p4_clock();
    start_ustime = p4_ustimer();
    if (me == 0) {
      (void) p4_send(type, left, buffer, lenbuf);
      msg = (int *) NULL;
      (void) p4_recv(&type, &right, (char **) &msg, &msg_len);
      p4_msg_free((char *) msg);
    }
    else {
      msg = (int *) NULL;
      (void) p4_recv(&type, &right, (char **) &msg, &msg_len);
      (void) p4_send(type, left, (char *) msg, msg_len);
      p4_msg_free((char *) msg);
    }
    used = p4_clock() - start;
    used_ustime = p4_ustimer() - start_ustime;

    if (used > 0)
      rate = 1.0e-3 * (double) (nproc * lenbuf) / (double) used;
    else
      rate = 0.0;
    if (me == 0)
      (void) printf("len=%d bytes, used=%d ms, used_us=%d rate=%f Mb/s\n",
		    lenbuf, used, used_ustime, rate);
    
    lenbuf *= 2;
  }

  if (me == 0)
    (void) p4_shfree(buffer);

}

double ranf()
/* Returns ran # uniform in (0,1) ... probably rather bad statistics. */
{
  static unsigned long seed = 54321;

  seed = seed * 1812433253 + 12345;
  return (seed & 0x7fffffff) * 4.6566128752458e-10;
}

static void RandList(lo, hi, list, n)
     int lo, hi, *list, n;
/*
  Fill list with n random integers between lo & hi inclusively
*/
{
  int i, ran;
  double dran;

  for (i=0; i<n; i++) {
    dran = ranf();
    ran = lo + (int) (dran * (double) (hi-lo+1));
    if (ran < lo)
      ran = lo;
    if (ran > hi)
      ran = hi;
    list[i] = ran;
  }
}

static void Stress()
/*
  Stress the system by passing messages between a randomly selected
  list of nodes
*/
{
#define N_LEN 10
  static int len[N_LEN] = {0,1,2,4,8,4096,8192,16384,32768,65536};
  int me = p4_get_my_id();
  int nproc = p4_num_total_ids();
  int zero = 0;
  int type, lenbuf, i, j, from, to;
  int *list_i, *list_j, *list_n;
  char *buffer;
  int n_stress, mod, *msg, msg_len;

  type = 6;
  if (me == 0) {
    (void) printf("\nStress test ... randomly exchange messages\n-----------");
    (void) printf("\n\nInput no. of messages: ");
    (void) fflush(stdout);
  }
  n_stress = GlobalReadInteger();
  if ( (n_stress <= 0) || (n_stress > 100000) )
    n_stress = 1000;
  p4_dprintfl(00,"n_stress=%d\n",n_stress);

  lenbuf = n_stress * sizeof(int);

  if (!(buffer = p4_shmalloc((unsigned) len[N_LEN-1])))
    p4_error("Stress: failed to allocate buffer", len[N_LEN-1]);

  type = 7;
  if (me == 0) { /* Make random list of pairs and message lengths */
    if (!(list_i = (int *) p4_shmalloc((unsigned) lenbuf)))
      p4_error("Stress: failed to allocate list_i",lenbuf);
    if (!(list_j = (int *) p4_shmalloc((unsigned) lenbuf)))
      p4_error("Stress: failed to allocate list_j",lenbuf);
    if (!(list_n = (int *) p4_shmalloc((unsigned) lenbuf)))
      p4_error("Stress: failed to allocate list_n",lenbuf);

    RandList((int) 0, nproc-1, list_i, n_stress);
    RandList((int) 0, nproc-1, list_j, n_stress);
    RandList((int) 0, N_LEN-1, list_n, n_stress);
    for (i=0; i<n_stress; i++)
      list_n[i] = len[list_n[i]];
    p4_broadcastx(type, (char *) list_i, lenbuf, P4INT);
    p4_broadcastx(type, (char *) list_j, lenbuf, P4INT);
    p4_broadcastx(type, (char *) list_n, lenbuf, P4INT);
  }
  else {
    list_i = (int *) NULL;
    (void) p4_recv(&type, &zero, (char **) &list_i, &msg_len);
    list_j = (int *) NULL;
    (void) p4_recv(&type, &zero, (char **) &list_j, &msg_len);
    list_n = (int *) NULL;
    (void) p4_recv(&type, &zero, (char **) &list_n, &msg_len);
  }

  type = 8;

  j = 0;
  mod = (n_stress-1)/10 + 1;
  for (i=0; i < n_stress; i++) {

    from   = list_i[i];
    to     = list_j[i];
    lenbuf = list_n[i];

    /* P4 can send to self 
    if (from == to)
      continue; */

    if ( (me == 0) && (j%mod == 0) ) {
      (void) printf("Stress: test=%ld: from=%ld, to=%ld, len=%ld\n",
		    i, from, to, lenbuf);
      (void) fflush(stdout);
    }

    j++;  /* Needed when skipping send to self */

    if (from == me)
      (void) p4_send(type, to, buffer, lenbuf);

    if (to == me) {
      msg = (int *) NULL;
      (void) p4_recv(&type, &from, (char **) &msg, &msg_len);
      p4_msg_free((char *) msg);
      if (msg_len != lenbuf)
	p4_error("Stress: invalid message length on receive",lenbuf);
    }
  }

  (void) p4_shfree(buffer);
  if (me == 0) {
    (void) p4_shfree((char *) list_n);
    (void) p4_shfree((char *) list_j);
    (void) p4_shfree((char *) list_i);
  }
  else {
    (void) p4_msg_free((char *) list_n);
    (void) p4_msg_free((char *) list_j);
    (void) p4_msg_free((char *) list_i);
  }
}

static int GlobalReadInteger()
/*
  Process zero reads an integer from stdin and broadcasts
  to everyone else
*/
{
  int value, *msg, msg_len, type=999,zero=0;

  if (p4_get_my_id() == 0) {
    if (scanf("%d", &value) != 1)
      p4_error("failed reading integer value from stdin", -1);

    p4_broadcastx(type, (char *) &value, sizeof value,P4INT);
  }
  else {
    msg = (int *) NULL;
    (void) p4_recv(&type, &zero, (char **) &msg, &msg_len);
    value = *msg;
    p4_msg_free((char *) msg);
  }
  return value;
}

static int CompareVectors(n, a, b)
     int n;
     double *a, *b;
/*
  Return the no. of differences in two vectors allowing for
  numerical roundoff.
*/
{
#define ABS(a)   (((a)>=0 ) ? (a) : -(a))
  int nerrs = 0;
  double diff;

  while (n--) {
    diff = *a++ - *b++;
    if (ABS(diff) > 1.0e-8)
      nerrs++;
  }
  
  return nerrs;
}

static void Globals()
/*
  Test out functioning of the global operations.
*/
{
  int nproc = p4_num_total_ids();
  int me = p4_get_my_id();
  int n, i, start, used, nerrs;
  double *a, *b, rate;

#define DO(string, op) \
  start = p4_clock(); \
  if (p4_global_op(33, (char *) a, n, sizeof(double), op, P4DBL)) \
    p4_error("p4_global_op failed",n); \
  used = p4_clock()-start; \
  rate = (used>0) ? n/(1.0e+3 * used) : 0.0; \
  nerrs = CompareVectors(n, a, b); \
  if (me == 0) \
    (void) printf("%s, len=%d, used=%d ms, rate=%f Mop/s, nerrs=%d\n",\
                   string, n, used, rate, nerrs);

  if (me == 0) {
    (void) printf("\nGlobal operations test\n----------------------");
    (void) printf("\n\nInput vector length ");
    (void) fflush(stdout);
  }
  n = GlobalReadInteger();
  if ( (n < 0) || (n > 1000000) )
    n = 1000;

  if (!(a = (double *) p4_shmalloc((unsigned) (n*sizeof(double)))))
    p4_error("failed to create work space (a)",n);
  if (!(b = (double *) p4_shmalloc((unsigned) (n*sizeof(double)))))
    p4_error("failed to create work space (b)",n);

  /* Summation */

  for (i=0; i<n; i++) {
    a[i] = i+me;
    b[i] = nproc*i + (nproc*(nproc-1))/2;
  }
  DO("Summation", p4_dbl_sum_op);

  /* Maximum */

  for (i=0; i<n; i++) {
    a[i] = i+me;
    b[i] = i+nproc-1;
  }
  DO("Maximum", p4_dbl_max_op);

  /* Abs Maximum */

  for (i=0; i<n; i++) {
    a[i] = i+me - n/2;
    b[i] = MAX(n/2-i, i+nproc-1-n/2);
  }
  DO("Abs Maximum", p4_dbl_absmax_op);

  /* Tidy up */

  p4_shfree((char *) b);
  p4_shfree((char *) a);
}


void synchronize(type)
     int type;
/*
  Processes block until all have checked in with process 0
  with a message of specified type .. a barrier.
*/
{
  int me = p4_get_my_id();
  int nproc = p4_num_total_ids();
  int zero = 0;
  int *msg;
  int msg_len, node, dummy = type;

  if (me == zero) {
    for (node=1; node<nproc; node++){       /* Check in */
      msg = (int *) NULL;
      if (p4_recv(&type, &node, (char **) &msg, &msg_len))
	p4_error("synchronize: recv 1 failed", (int) msg);
      p4_msg_free((char *) msg);
    }
    if (p4_broadcast(type, (char *) &dummy, sizeof dummy))
      p4_error("synchronize: broadcast failed",type);
  }
  else {
    if (p4_send(type, zero, (char *) &me, sizeof me))
      p4_error("synchronize: send failed", type);
    msg = (int *) NULL;
    if (p4_recv(&type, &zero, (char **) &msg, &msg_len))
      p4_error("synchronize: recv 2 failed", (int) msg);
    p4_msg_free((char *) msg);
  }
}    
SHAR_EOF
if test -f 'vec_master.c'
then
	echo shar: over-writing existing file "'vec_master.c'"
fi
cat << \SHAR_EOF > 'vec_master.c'
#include "p4.h"
#include "sr_user.h"
    
#define MAX_VECLEN 10000

int msg[MAX_VECLEN];

main(argc,argv)
int argc;
char **argv;
{
    int nslaves;
    int type, size, id, from;
    int my_id;
    char *incoming;
    int done;
    int i, veclen, count;
    int starttime, endtime;
    p4_usc_time_t start_ustime, end_ustime, rollover;

    p4_initenv(&argc,argv);
    p4_create_procgroup();
    
    nslaves = p4_num_total_ids() - 1;
    printf("number of slaves = %d\n",nslaves);
    my_id = p4_get_my_id();
    rollover = p4_usrollover();
    printf("rollover=%d\n",rollover);
    
    done = FALSE;
    while (!done)
    {
	printf("vector length: ");
	scanf("%d",&veclen);
	if (veclen > MAX_VECLEN)
	{
	    printf("too big;  using %d\n",MAX_VECLEN);
	    veclen = MAX_VECLEN;
	}
	for (i=0; i<veclen; i++)
	    msg[i] = i;

	printf("times around loop (or 0 for end): ");
	scanf("%d",&count);
	
	if (count == 0)
	    done = TRUE;
	else
	{
	    starttime = p4_clock();
	    start_ustime = p4_ustimer();
	    while (count > 0)
	    {
		p4_sendx(DATA, 1, msg, veclen*sizeof(int), P4INT);
		type = -1;
		from = -1;
		incoming = NULL;
		p4_recv(&type, &from, &incoming, &size);
		compare_vec(msg,incoming,veclen);
		p4_msg_free(incoming);
		count--;
	    }
	    end_ustime = p4_ustimer();
	    endtime = p4_clock();
	    printf("time %d milliseconds\n",endtime-starttime);
	    printf("time %d microseconds\n",end_ustime-start_ustime);
	}
    }

    p4_send(END, 1, msg, 0);
    type = -1;
    from = -1;
    incoming = NULL;
    p4_recv(&type, &from, &incoming, &size);
    p4_msg_free(incoming);
    p4_wait_for_end();
    printf("master exiting normally\n");
}

compare_vec(a,b,len)
int a[], b[];
int len;
{
    int i;

    for (i=0; i < len; i++)
    {
	if (a[i] != b[i])
	    printf("a[%d] = %d, b[%d] = %d\n",i,a[i],i,b[i]);
    }
}
	    
SHAR_EOF
if test -f 'vec_slave.c'
then
	echo shar: over-writing existing file "'vec_slave.c'"
fi
cat << \SHAR_EOF > 'vec_slave.c'
#include "p4.h"
#include "sr_user.h"
    
slave()	
{
    int nslaves;
    int done;
    int type, from, size;
    int next;
    int my_id;
    char *incoming;
    
    my_id = p4_get_my_id();
    nslaves = p4_num_total_ids() - 1;
    
    if (my_id == nslaves)
        next = 0;
    else
	next = my_id + 1;
    
    done = FALSE;
    while (!done)
    {
	type = -1;
	from = -1;
	incoming = NULL;
	p4_recv(&type,&from, &incoming, &size);
	p4_sendx(type, next, incoming, size, P4INT);
	if (type == END)
	    done = TRUE;
	p4_msg_free(incoming);
    }
}


SHAR_EOF
if test -f 'runsock'
then
	echo shar: over-writing existing file "'runsock'"
fi
cat << \SHAR_EOF > 'runsock'
mexec -t "(2,2)" -f "0 1 sr_test $1 $2 $3 "
SHAR_EOF
chmod +x 'runsock'
if test -f 'nextest.pg'
then
	echo shar: over-writing existing file "'nextest.pg'"
fi
cat << \SHAR_EOF > 'nextest.pg'
local 0
#locke.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
killdeer.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
eris.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
fulmar.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
hilda.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
loon.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
osprey.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
pelican.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
macromol.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
lucky.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
plover.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
raven.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
scoter.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#skua.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#tern.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#jaeger.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#tango.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#sunset.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#rainbow.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#shopper.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#kafka.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#grivator.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#auklet.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#cupie.mcs.anl.gov 1 /private/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
SHAR_EOF
if test -f 'suntest.pg'
then
	echo shar: over-writing existing file "'suntest.pg'"
fi
cat << \SHAR_EOF > 'suntest.pg'
local 0
##godzilla.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
juju.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
skeeve.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#condor.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
##merlin.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#chartres.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#einsicht.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#thales.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#jadoube.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#vulcan.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#canopus.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#altair.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#dalek.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#ezra.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#shark.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#lutra.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
#procyon.mcs.anl.gov 1 /tmp_mnt/Net/sparky/sparky1/lusk/p4-1.0/messages/systest
SHAR_EOF
cd ..
if test ! -d 'messages_f'
then
	mkdir 'messages_f'
fi
cd 'messages_f'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains some simple examples and test programs for the
Fortran interface to p4.

sr   - This program sends a user-entered message around a ring of
       processes.  It is terminated by typing the end-of file
       character.

sr1  - This program prompts for a message size and number of times
       to circulate the message around the ring.

simpglob - Tests global operations.

systest - A preliminary installation-testing program.

*.pg - The procgroup files necessary for running the above programs on
       various configurations of machines.
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIB = $(LIBDIR)/libp4.a
FORTLIBDIR = $(P4_HOME_DIR)/lib_f
FORTLIB = $(FORTLIBDIR)/libp4_f.a

MAIN = $(FORTLIBDIR)/p4_fmain.o

LIBS =  $(FORTLIB) $(LIB) $(MDEP_LIBS)

default:$(P)
	make $(MFLAGS) systest simpglob

all:$(P)
	make $(MFLAGS) systest sr_master sr1_master
	make $(MFLAGS) sr_slave sr1_slave

systest: systest.o $(FORTLIB) $(LIB)
	$(FC) $(MDEP_FFLAGS) -o systest systest.o $(LIBS)
	
simpglob: simpglob.o $(FORTLIB) $(LIB)
	$(FC) $(MDEP_FFLAGS) -o simpglob simpglob.o $(LIBS)
	
sr_master: sr_master.o sr_slave.o  $(FORTLIB) $(LIB)
	$(FC) $(MDEP_FFLAGS) -o sr_master sr_master.o sr_slave.o $(LIBS)
	
sr_slave: sr_slave.o  $(FORTLIB)  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o sr_slave $(MAIN) sr_slave.o $(LIBS)

sr1_master: sr1_master.o sr1_slave.o $(FORTLIB)  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o sr1_master sr1_master.o sr1_slave.o $(LIBS)
	
sr1_slave: sr1_slave.o  $(FORTLIB)  $(LIB)
	$(FC) $(MDEP_FFLAGS) -o sr1_slave $(MAIN) sr1_slave.o $(LIBS)

realclean: clean
	$(RM) -f systest simpglob sr_master sr_slave \
		sr1_master sr1_slave Makefile p4f.h
SHAR_EOF
if test -f 'simpglob.f'
then
	echo shar: over-writing existing file "'simpglob.f'"
fi
cat << \SHAR_EOF > 'simpglob.f'
      program systest

      include 'p4f.h'

      call p4init()
      if (p4myid() .eq. 0) then
          call p4crpg()
      endif
      call fslave()
      call p4cleanup()
      print *,'exiting pgm'
      end


      subroutine fslave()

      include 'p4f.h'

      character*40 buffer
      integer ASIZE
      parameter (ASIZE = 10)
      double precision  a(ASIZE)
      integer procid, itype, iasize, idblsize, ip4dbl, rc

      procid = p4myid()

      print 200,'slave ',procid,' has started'
 200  format(a,i2,a)
      call p4flush

      do 10 i = 1,ASIZE
          a(i) = i
 10   continue

      itype = 44
      iasize = ASIZE
      idblsize = 8
      ip4dbl = P4DBL
      call p4globop(itype,a,iasize,idblsize,p4dblsumop,ip4dbl,rc)

      do 20 i = 1,ASIZE
          print 300,a(i)
          call p4flush
 20   continue
300   format(f4.0)

      print 500,'slave ',procid,' is exiting'
 500  format(a,i2,a)
      call p4flush

      end
SHAR_EOF
if test -f 'sr1_anagram.pg'
then
	echo shar: over-writing existing file "'sr1_anagram.pg'"
fi
cat << \SHAR_EOF > 'sr1_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/sr1_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/sr1_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/sr1_slave
#anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_newf/sr1_slave
SHAR_EOF
if test -f 'sr1_master.f'
then
	echo shar: over-writing existing file "'sr1_master.f'"
fi
cat << \SHAR_EOF > 'sr1_master.f'
      program master

      include 'p4f.h'

      integer i,slaves,type,from,retcde,recvlen,buflen,loops,time
      character*500 buffer
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      call p4init()
      call p4crpg()

      slaves = p4ntotids() - 1
      length = 0
      buflen = 0

      do 10 i = 1,slaves
         call p4sendr(TAGCNT,i,buffer,length,retcde)
10    continue

20    print *,'How long is the message: '
      read (*,*,end=50) buflen
      buflen = MIN(500,buflen)

25    print *,'How many times around the loop? '
      read (*,*,end=50) loops

      length = buflen 
      time = p4clock()
      do 45 j=1,loops

C send in reverse order of the slaves
      call p4send(TAGDAT,slaves,buffer,length,retcde)
      buffer = ' ' 
      type = TAGDAT
      from = -1
      call p4recv(type,from,buffer,buflen,recvlen,type,retcde)

      print *,'Loop number = ',j
      print 101,'MASTER receives from = ',from,' length = ',recvlen
101   format(a,i2,a,i6,/)
      length = recvlen 
45    continue 
      oldtime = time
      time = p4clock()
      print *,'Time used: ',time-oldtime, ' milliseconds'
      length = 0
50    continue

      do 60 i = 1,slaves
         call p4sendr(TAGEND,i,buffer,buflen,retcde)
60    continue

      call p4cleanup()
      print *,'Master exiting normally'
      end
SHAR_EOF
if test -f 'sr1_slave.f'
then
	echo shar: over-writing existing file "'sr1_slave.f'"
fi
cat << \SHAR_EOF > 'sr1_slave.f'
      subroutine fslave()

      include 'p4f.h'

      character*500 buffer
      integer type, from, next, done, procid, length
      integer numsl, retcde, recvlen
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      numsl = p4ntotids() - 1
      procid = p4myid()

      print 200,'slave ',procid,' has started'
 200  format(a,i2,a,/)
      call p4flush()

      if (procid .eq. 1) then
         next = 0
      else
         next = procid - 1
      endif

      print 201,'slave ',procid,' next = ',next
 201  format(a,i2,a,i2,/)
      call p4flush()

      length = 0
      from = -1
      type = TAGCNT
      call p4recv(type,from,buffer,length,recvlen,retcde)
      done = 0

50    if (done .ne. 0) goto 100

         buffer = ' '
         length = 500
         from = -1
         type = -1
         call p4recv(type,from,buffer,length,recvlen,retcde)
         if (type .eq. TAGEND) then
            done = 1
         else
            print *,'SLAVE ',procid,' sending msg to ',next
            call p4flush()
            call p4send(TAGDAT,next,buffer,recvlen,retcde)
         endif
         goto 50


100   continue

      end
SHAR_EOF
if test -f 'sr_anagram.pg'
then
	echo shar: over-writing existing file "'sr_anagram.pg'"
fi
cat << \SHAR_EOF > 'sr_anagram.pg'
local 0
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_f/sr_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_f/sr_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_f/sr_slave
anagram.mcs.anl.gov 1 /Net/anagram/anagram8/lusk/p4/messages_f/sr_slave


SHAR_EOF
if test -f 'sr_cube.pg'
then
	echo shar: over-writing existing file "'sr_cube.pg'"
fi
cat << \SHAR_EOF > 'sr_cube.pg'
# local 0
backend 2 /home/sun2/u1/lusk/p4.node/messages_f/slave
SHAR_EOF
if test -f 'sr_ibm.pg'
then
	echo shar: over-writing existing file "'sr_ibm.pg'"
fi
cat << \SHAR_EOF > 'sr_ibm.pg'
local 0
ibm.mcs.anl.gov 1 /Net/sparky/sparky1/lusk/ibm/p4/messages_newf/slave


SHAR_EOF
if test -f 'sr_local.pg'
then
	echo shar: over-writing existing file "'sr_local.pg'"
fi
cat << \SHAR_EOF > 'sr_local.pg'
local 5



SHAR_EOF
if test -f 'sr_master.f'
then
	echo shar: over-writing existing file "'sr_master.f'"
fi
cat << \SHAR_EOF > 'sr_master.f'
      program master

      include 'p4f.h'

      integer i,slaves,type,from,retcde,recvlen,buflen
      character*40 buffer
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      call p4init()
      call p4crpg()

      slaves = p4ntotids() - 1
      length = 0
      buflen = 40

      do 10 i = 1,slaves
         call p4sendr(TAGCNT,i,buffer,length,retcde)
10    continue

20    print *,'Type a string: '
      read (*,99,end=50) buffer
99    format(a40)

      do 30 length=40,1,-1
         if(buffer(length:length) .ne. ' ') goto 40
30    continue
      length = 0
40    continue

      call p4send(TAGDAT,1,buffer,length,retcde)
      buffer = ' '
      type = TAGDAT
      from = -1
      call p4recv(type,from,buffer,buflen,recvlen,retcde)

      print *,'MASTER receives= ',buffer,' from ',from
      length = 0
      goto 20
50    continue

      do 60 i = 1,slaves
         call p4sendr(TAGEND,i,buffer,buflen,retcde)
60    continue

      call p4cleanup()
      print *,'Master exiting normally'
      end
SHAR_EOF
if test -f 'sr_slave.f'
then
	echo shar: over-writing existing file "'sr_slave.f'"
fi
cat << \SHAR_EOF > 'sr_slave.f'
      subroutine fslave()

      include 'p4f.h'

      character*40 buffer
      integer type, from, next, done, procid, length, buflen
      integer numsl, retcde, recvlen
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      numsl = p4ntotids() - 1
      procid = p4myid()
      buflen = 40

      print 200,'slave ',procid,' has started'
 200  format(a,i2,a)
      call p4flush

      if (procid .eq. numsl) then
         next = 0
      else
         next = procid + 1
      endif

      print 201,'slave ',procid,' next = ',next
 201  format(a,i2,a,i2)
      call p4flush

      length = 40
      from = -1
      type = TAGCNT
      call p4recv(type,from,buffer,length,recvlen,retcde)
      done = 0

50    if (done .ne. 0) goto 100

         buffer = ' '
         length = 40
         from = -1
         type = -1
         call p4recv(type,from,buffer,length,recvlen,retcde)
         if (type .eq. TAGEND) then
            done = 1
         else
                print *,'SLAVE ',procid,' sending msg to ',next
                call p4flush
            call p4send(TAGDAT,next,buffer,recvlen,retcde)
         endif
         goto 50

100   continue

      end
SHAR_EOF
if test -f 'systest.f'
then
	echo shar: over-writing existing file "'systest.f'"
fi
cat << \SHAR_EOF > 'systest.f'
      program systest

      include 'p4f.h'

      call p4init()
      if (p4myid() .eq. 0) then
          call p4crpg()
	  call fmaster()
      else
	  call fslave()
      endif
      call p4cleanup()
      print *,'mainline exiting normally'
      end


      subroutine fmaster()

      include 'p4f.h'

      integer i,slaves,type,from,retcde,recvlen,buflen
      character*40 buffer
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      print 11,'Entering fmaster'
11    format(a)
      slaves = p4ntotids() - 1
      length = 0
      buflen = 40

      do 10 i = 1,slaves
         call p4sendr(TAGCNT,i,buffer,length,retcde)
10    continue

20    print *,'Enter a string: '
      read (*,99,end=50) buffer
99    format(a40)

      do 30 length=40,1,-1
         if(buffer(length:length) .ne. ' ') goto 40
30    continue
      length = 0
40    continue

      call p4send(TAGDAT,1,buffer,length,retcde)
      buffer = ' '
      type = TAGDAT
      from = -1
      call p4recv(type,from,buffer,buflen,recvlen,retcde)

      print *,'MASTER receives from=',from,' buffer=',buffer
      length = 0
      goto 20
50    continue

      do 60 i = 1,slaves
         call p4sendr(TAGEND,i,buffer,buflen,retcde)
60    continue

      print *,'Master exiting normally'
      end


      subroutine fslave()

      include 'p4f.h'

      character*40 buffer
      integer type, from, next, done, procid, length, buflen
      integer numsl, retcde, recvlen
      integer TAGCNT, TAGDAT, TAGEND
      parameter (TAGCNT = 10)
      parameter (TAGDAT = 20)
      parameter (TAGEND = 30)

      numsl = p4ntotids() - 1
      procid = p4myid()
      buflen = 40

C     print 200,'slave ',procid,' has started'
C200  format(a,i2,a)
C     call p4flush

      if (procid .eq. numsl) then
         next = 0
      else
         next = procid + 1
      endif

C     print 201,'slave ',procid,' next = ',next
C201  format(a,i2,a,i2)
C     call p4flush

      length = 40
      from = -1
      type = TAGCNT
      call p4recv(type,from,buffer,length,recvlen,retcde)
      done = 0

50    if (done .ne. 0) goto 100

         buffer = ' '
         length = 40
         from = -1
         type = -1
         call p4recv(type,from,buffer,length,recvlen,retcde)
         if (type .eq. TAGEND) then
            done = 1
         else
            call p4send(TAGDAT,next,buffer,recvlen,retcde)
         endif
         goto 50

100   continue

      end
SHAR_EOF
if test -f 'procgroup'
then
	echo shar: over-writing existing file "'procgroup'"
fi
cat << \SHAR_EOF > 'procgroup'
local 0
anagram 1 /Net/anagram/anagram3/rbutler/p4-1.0/messages_f/simpglob
#ree 1 /home/rbutler/8h/p4t03/messages_f/simpglob
SHAR_EOF
cd ..
if test ! -d 'monitors'
then
	mkdir 'monitors'
fi
cd 'monitors'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'

These are examples of shared-memory parallel programming using monitors.
The programs are:

  addtwo.gs     - add two vectors using a getsub monitor
  addtwo.ask    - add two vectors using an askfor monitor
  grid.shmem    - solve Dirichlet problem in plane
  gridlog.shmem - same as grid.shmem instrumented with alog
  adquad        - adaptive quadrature

Each of them can be run by redirecting input from the appropriate .data file.
The grid.shmem and adquad programs can also be run interactively.

Note that these examples will run on a uniprocessor with one process, but
p4 does not support shared memory among separate processes on any of the
supported uniprocessors (such as SUN, ATT_3B2, or IBM_RS_6000).
SHAR_EOF
if test -f 'addtwo.ask.c'
then
	echo shar: over-writing existing file "'addtwo.ask.c'"
fi
cat << \SHAR_EOF > 'addtwo.ask.c'
#include "p4.h"

#define MAXLEN 500
#define MAXPROCS 256

struct globmem {
    int length;
    int a[MAXLEN], b[MAXLEN], c[MAXLEN];
    int num_added[MAXPROCS];
    int sub;
    int slave_id;
    p4_lock_t go_lock;
    p4_askfor_monitor_t askfor;
} *glob;


int getprob(v)			/* return next available subscript */
P4VOID *v;
{
int rc = 1;   /* any non-zero means NO problem found */
int *p;

    p = (int *) v;
    if (glob->sub < glob->length)
    {
	*p = glob->sub++;
	rc = 0;    /* FOUND a problem */
    }
    return(rc);
}

P4VOID reset()
{
}

slave()
{
    work();
}

main(argc,argv)
int argc;
char **argv;
{
int i, j, n, start, end;

    p4_initenv(&argc,argv);   /* args not used but passed for compat */

    glob = (struct globmem *) p4_shmalloc(sizeof(struct globmem));

    glob->sub = 0;
    glob->slave_id = 0;
    p4_lock_init(&(glob->go_lock));
    p4_askfor_init(&(glob->askfor));

    /* read in the length and the two vectors */
    scanf("%d",&glob->length);
    for (i = 0; i < glob->length; i++)
	scanf("%d",&glob->a[i]);
    for (i = 0; i < glob->length; i++)
	scanf("%d",&glob->b[i]);
    
    p4_create_procgroup();

    start = p4_clock();
    work();
    end   = p4_clock();

    for (i = 0; i < glob->length;) 
    {
	for (j = 0; (j < 9) && (i < glob->length); j++)
	    printf("%d\t", glob->c[i++]);
	printf("\n");
    }
    for (i = 0, n = p4_num_total_ids(); i < n; i++)
	printf("num by %d = %d \n",i,glob->num_added[i]);

    p4_wait_for_end();
}

work()
{
int i, j, myid, nprocs, rc;

    /* lock/unlock acts like a barrier */
    p4_lock(&(glob->go_lock));
    p4_unlock(&(glob->go_lock));

    myid = p4_get_my_id();
    glob->num_added[myid] = 0;
    nprocs = p4_num_total_ids();
    while (p4_askfor(&(glob->askfor),nprocs,getprob,(P4VOID *)&i,reset) == 0)
    {
	glob->c[i] = glob->a[i] + glob->b[i];
	glob->num_added[myid]++;
    }
}
SHAR_EOF
if test -f 'addtwo.data'
then
	echo shar: over-writing existing file "'addtwo.data'"
fi
cat << \SHAR_EOF > 'addtwo.data'
5
1
2
3
4
5
1
2
3
4
5
SHAR_EOF
if test -f 'addtwo.data.big'
then
	echo shar: over-writing existing file "'addtwo.data.big'"
fi
cat << \SHAR_EOF > 'addtwo.data.big'
200
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
1
2
3
4
5
6
7
8
9
10
SHAR_EOF
if test -f 'addtwo.gs.c'
then
	echo shar: over-writing existing file "'addtwo.gs.c'"
fi
cat << \SHAR_EOF > 'addtwo.gs.c'
#include "p4.h"

#define MAXLEN 500
#define MAXPROCS 256


struct globmem {
    int length;
    int a[MAXLEN], b[MAXLEN], c[MAXLEN];
    int num_added[MAXPROCS];
    p4_barrier_monitor_t barrier;
    p4_getsub_monitor_t getsub;
} *glob;


slave()
{
    work();
}

main(argc,argv)
int  argc;
char **argv;
{
int i, j, n;

    p4_initenv(&argc,argv);

    glob = (struct globmem *) p4_shmalloc(sizeof(struct globmem));

    p4_barrier_init(&(glob->barrier));
    p4_getsub_init(&(glob->getsub));

    /* read in the length and the two vectors */
    scanf("%d",&glob->length);
    for (i = 0; i < glob->length; i++)
	scanf("%d",&glob->a[i]);
    for (i = 0; i < glob->length; i++)
	scanf("%d",&glob->b[i]);
    
    p4_create_procgroup();
    work();
    /* print the answer */
    for (i = 0; i < glob->length;) 
    {
	for (j = 0; (j < 9) && (i < glob->length); j++)
	    printf("%d\t", glob->c[i++]);
	printf("\n");
    }
    for (i = 0, n = p4_num_total_ids(); i < n; i++)
	printf("num by %d = %d \n",i,glob->num_added[i]);

    p4_wait_for_end();
}

work()
{
int i, myid, nprocs;

    myid = p4_get_my_id();
    glob->num_added[myid] = 0;
    nprocs = p4_num_total_ids();
    p4_barrier(&(glob->barrier),nprocs);
    p4_getsub(&(glob->getsub),&i,glob->length - 1,nprocs);
    while (i >= 0) {
	glob->c[i] = glob->a[i] + glob->b[i];
	glob->num_added[myid]++;
	p4_getsub(&(glob->getsub),&i,glob->length - 1,nprocs);
    }
}
SHAR_EOF
if test -f 'adquad.c'
then
	echo shar: over-writing existing file "'adquad.c'"
fi
cat << \SHAR_EOF > 'adquad.c'
#include <stdio.h>
#include <math.h>
#include "p4.h"
    
    /*  NOTE: This program contains some features which are more
	general than required for this specific task.  Please
	remember that we hope to use this as a more-or-less
	prototypical use of trees in numeric algorithms.  For
	example, the definition of tree_node includes "first_child"
	and "sibling" pointers, although they are never actually
	used (and, the tree could in any event just be
	a binary tree, rather than a binary tree representing
	a general tree).  Our intent is to discuss the topics
	of "general trees", "representing general trees with
	binary trees", etc.  It is also the case that the
	use of queue_nodes is unnecessary; you could just
	add a link field to the tree_nodes and queue the
	ready tree_nodes in the pool.  
	*/
    
    /* this structure gives the format of a node in the tree
       representing the gradual subdivision of the interval.
       Each node represented a portion of the interval from
       "xl" to "xr".
       */

int queue_node();

struct tree_node {
    double xl;		/* the left bound */
    double xm;		/* the midpoint */
    double xr;		/* the right bound */
    double yl,ym,yr;	/* values of the function */
    double integral;	/* computed value of the integral */
    int status;	        /* 0->not subdivided;
			   1->one side completed
			   2->both sides completed
			   
			   another way to think of this field is
			   as a count of the number of subcalculations
			   that have been completed
			   */
    struct tree_node *parent; /* pointer to the parent node */
    struct tree_node *first_child,*sibling;
    p4_lock_t t_lock;
};

struct queue_node {
    struct queue_node *next; /* link to next node in pool or q_avail */
    struct tree_node *node;  /* when node is in pool, this field points
				to a node in the tree that has been
				queued in the "pool" of work.
				*/
};

/* variables
   
   tree     - the tree representing the current state
   of the computation
   
   pool     - the pool of problems.  Initially, only one node
   will be in the pool, and it will represent the
   entire interval
   
   q_avail  - avail-list for queue nodes
   
   t_avail  - avail-list for tree nodes
   
   numprocs - number of processes to participate in the
   calculation
   
   normdiff - the error you are willing to tolerate in a
   unit interval
   */

struct globmem {
    
    p4_lock_t TAVL;
    struct tree_node *t_avail;  /* list of available tree_nodes */
    struct tree_node *tree;     /* the tree being built */
    
    p4_lock_t QAVL;
    struct queue_node *q_avail; /* list of available queue_nodes */
    struct queue_node *pool;    /* pool of available work */
    
    int numprocs;		/* number of processes */
    double normdiff;		/* absolute amount of tolerable error */
    
    p4_askfor_monitor_t MO;
} *glob;


slave()
{
    work();
}

main(argc,argv)
int argc;
char **argv;
{
    extern slave();
    char c;
    double r;
    int i,j;
    long stime,etime;
    double func();
    struct tree_node *t_node;
    struct tree_node *alloc_tree_node();
    
    
    p4_initenv(&argc,argv);
    glob = (struct globmem *) p4_shmalloc(sizeof(struct globmem));
    
    glob->t_avail = NULL;
    glob->q_avail = NULL;
    p4_lock_init(&(glob->TAVL));
    p4_lock_init(&(glob->QAVL));
    p4_askfor_init(&(glob->MO));
    
    t_node = alloc_tree_node(); 
    glob->tree = t_node;
    
    printf("left boundary: ");
    scanf("%lf",&(t_node->xl));
    printf("right boundary: ");
    scanf("%lf",&(t_node->xr));
    
    if (t_node->xr > 15.0)
    {
	printf("right boundary too big; try 15.0\n");
	exit(0);
    }
    
    /* set up original node - the midpoint, function
       values, and first approx of integral must be
       stored in the node
       */
    t_node->xm = (t_node->xl + t_node->xr) / 2.0;
    
    t_node->yl = func(t_node->xl);
    t_node->ym = func(t_node->xm);
    t_node->yr = func(t_node->xr);
    
    t_node->integral = (t_node->xr - t_node->xl) *
	(t_node->yl +
	 t_node->yr +
	 (4 * t_node->ym)) / 6.0;
    
    t_node->status = 0;
    t_node->parent = NULL;
    p4_lock_init(&(t_node->t_lock));
    
    /* stack the single node */
    p4_update(&(glob->MO),queue_node,(P4VOID *) t_node);
    
    printf("'allowable difference' for a unit: ");
    scanf("%lf",&(glob->normdiff));
    printf("number of processes: ");
    scanf("%d",&(glob->numprocs));
    
    stime = p4_clock();		/* note time includes process creation */
    
    /* create the slave processes */
    for (i=1; i < glob->numprocs; i++) 
	p4_create(slave);
    
    /* join the slaves in processing nodes in the pool */
    work(); 
    
    etime = p4_clock();
    
    printf("integral = %lf\n",glob->tree->integral); 
    printf("time = %d milliseconds\n",(etime - stime));
    
    p4_wait_for_end();
}

int getprob(v)
P4VOID **v;
{
    int rc = 1;   /* any non-zero means NO problem found */
    struct queue_node **p;
    
    p = (struct queue_node **) v;
    if (glob->pool != NULL)
    {
	*p = glob->pool;
        glob->pool = glob->pool->next;
	rc = 0;    /* FOUND a problem */
    }
    return(rc);
}

P4VOID reset()
{
}

work()
{
    int rc;
    struct tree_node *t_node;
    struct queue_node *q_node;
    int num_done = 0;
    
    rc = p4_askfor(&(glob->MO),glob->numprocs,getprob,(P4VOID *)&q_node,reset);
    
    while (rc == 0)
    {
	num_done++;
	t_node = q_node->node;
	dealloc_queue_node(q_node);
	
	evaluate(t_node);   /* process the node */
	
	rc = p4_askfor(&(glob->MO),glob->numprocs,getprob,(P4VOID *)&q_node,reset);
    }
    p4_dprintfl(5,"exiting work, did %d\n",num_done);
}

/* evaluate processes a node, which may cause subnodes
   to be created and stacked.
   */

evaluate(n)
struct tree_node *n;
{
    double xlm;  /* midpoint of left interval */
    double xrm;  /* midpoint of right interval */
    double leftint; /* integral of left */
    double rightint; /* integral of right */
    double ylm,yrm;  /* evaluated function for the midpoints */
    struct tree_node *lch,*rch;  /* pointers to children */
    struct tree_node *p; 
    double diff;
    struct tree_node *alloc_tree_node();
    
    /* first calculate the next level of approximation to
       see whether we are close enough
       */
    xlm = (n->xl + n->xm) / 2;
    xrm = (n->xm + n->xr) / 2;
    ylm = func(xlm);
    yrm = func(xrm);
    leftint = (n->xm - n->xl) * ((n->yl + n->ym + (4 * ylm)) / 6);
    rightint = (n->xr - n->xm) * ((n->ym + n->yr + (4 * yrm)) / 6);
    
    
    diff = fabs(n->integral - (leftint + rightint));
    
    if (diff < (glob->normdiff / (n->xr - n->xl)))
    {
	
	/* keep the more accurate estimate, and process completion
	   of this node
	   */
	n->integral = leftint + rightint;
	postcomp(n);
    }
    else
    {
	/* build the left node and stack it in the
	   pool of work to do
	   */
	lch = alloc_tree_node();
	lch->xl = n->xl;
	lch->xm = xlm;
	lch->xr = n->xm;
	lch->yl = n->yl;
	lch->ym = ylm;
	lch->yr = n->ym;
	lch->integral = leftint;
	lch->status = 0;
	lch->parent = n;
	p4_lock_init(&(lch->t_lock));

	p4_update(&(glob->MO),queue_node,(P4VOID *) lch);
	
	/* we now process the right child */
	rch = alloc_tree_node();
	rch->xl = n->xm;
	rch->xm = xrm;
	rch->xr = n->xr;
	rch->yl = n->ym;
	rch->ym = yrm;
	rch->yr = n->yr;
	rch->integral = rightint;
	rch->status = 0;
	rch->parent = n;
	p4_lock_init(&(rch->t_lock));
	
	evaluate(rch);
    }
}


/* this routine handles the "completion" of a node,
   which involves storing the answer in the parent node,
   checking to see if this completes the parent, and processing
   completion of the parent (if required).  The nodes are
   returned to the avail list as they are removed from the
   leaves of the tree (except the root!)
   */
postcomp(n)
struct tree_node *n;
{
    struct tree_node *p;   /* pointer to parent */
    int stat;         /* status of the parent */
    
    if ((p = n->parent) != NULL)
    {
	p4_lock(&(p->t_lock));
	if (p->status == 0) 
	    p->integral = 0.000;
	p->integral += n->integral;
	
	stat = ++(p->status);
	p4_unlock(&(p->t_lock));
	
	dealloc_tree_node(n);
	
        if (stat == 2)  /* if parent is complete too */
	{
	    postcomp(p);
	}
    }
}

/* this is the function to integrate */
double func(x)
double x;
{
    int i;
    
    /* the attribute "static" assures that the values do not change
       between invocations of the function.  That is, a single static
       copy is allocated and referenced by all calls to the routine.
       */
    static int power = 30;
    static int firstcall = 1;
    static double factor;
    static double pi;
    double v;
    
    
    if (firstcall)
    {
	factor = 1;
	for (i=1; (i <= (power/2)); i++)
	    factor = factor * (1 - (.5 / i));
	
	pi = 4 * atan((double) 1);
	firstcall = 0;
    }
    
    return(pow(sin((double) (pi * x)),(double) power) / factor);
}

/*
  this routine allocates a tree_node from globally-shared
  memory.
  */
struct tree_node *alloc_tree_node()
{
    struct tree_node *node;
    
    p4_lock(&(glob->TAVL));
    if ((node = glob->t_avail) == NULL)
    {
	node = (struct tree_node *) p4_shmalloc(sizeof(struct tree_node));
	if (node == NULL)
        {
            printf("*** out of memory in alloc_tree_node ***\n");
            exit(3);
        }
    }
    else
    {
        glob->t_avail = node->sibling;
    }
    p4_unlock(&(glob->TAVL));
    return(node);
}

/*
  this routine frees a tree_node
  */
dealloc_tree_node(node)
struct tree_node *node;
{
    
    p4_lock(&(glob->TAVL));
    node->sibling = glob->t_avail;
    glob->t_avail = node;
    p4_unlock(&(glob->TAVL));
}

/*
  this routine allocates a queue_node from globally-shared
  memory.
  */
struct queue_node *alloc_queue_node()
{
    struct queue_node *node;
    
    p4_lock(&(glob->QAVL));
    if ((node = glob->q_avail) == NULL)
    {
	node = (struct queue_node *) p4_shmalloc(sizeof(struct queue_node));
	if (node == NULL)
        {
            printf("*** out of memory in alloc_queue_node ***\n");
            exit(3);
        }
    }
    else
    {
        glob->q_avail = node->next;
    }
    p4_unlock(&(glob->QAVL));
    return(node);
}

/*
  this routine frees a queue_node
  */
dealloc_queue_node(node)
struct queue_node *node;
{
    
    p4_lock(&(glob->QAVL));
    node->next = glob->q_avail;
    glob->q_avail = node;
    p4_unlock(&(glob->QAVL));
}

/*
  this node adds a tree_node to the pool of work.
  This involves allocating a queue_node and hooking
  it into the pool.  Because the pool is constantly
  being altered by all of the processes, this must
  be a monitor operation.
  */

int queue_node(t_node)
struct tree_node *t_node;
{
    struct queue_node *alloc_queue_node();
    struct queue_node *q_node;
    
    q_node = alloc_queue_node();
    q_node->node = t_node;
    q_node->next = glob->pool;
    glob->pool = q_node;
    return(1);			/* new problem */
}
SHAR_EOF
if test -f 'adquad.data'
then
	echo shar: over-writing existing file "'adquad.data'"
fi
cat << \SHAR_EOF > 'adquad.data'
0.0
1.0
0.0001
2
SHAR_EOF
if test -f 'grid.data'
then
	echo shar: over-writing existing file "'grid.data'"
fi
cat << \SHAR_EOF > 'grid.data'
5
100
100
100
SHAR_EOF
if test -f 'grid.shmem.c'
then
	echo shar: over-writing existing file "'grid.shmem.c'"
fi
cat << \SHAR_EOF > 'grid.shmem.c'
#include "p4.h"
#define ROWS 		200
#define COLUMNS 	200
struct globmem {
    double a[ROWS+2][COLUMNS+2];
    double b[ROWS+2][COLUMNS+2];
    int st[ROWS+2], pq[ROWS+1];
    int pqbeg, pqend, goal, nproc, rows, columns;
    p4_askfor_monitor_t MO;
    p4_barrier_monitor_t BA;
} *glob;

double avggrid();
double avgbnd();

slave()
{
    work('s');
}

phi(x,y)			/* The function on the boundary */
int x,y;
{
      return((x * x) - (y * y) + (x * y));   
}

main(argc,argv)
int argc;
char **argv;
{
    int i;
    int timestart, timeend;
    p4_usc_time_t start_ustime, end_ustime;
    double avg;

    p4_initenv(&argc,argv);

    glob = (struct globmem *) p4_shmalloc(sizeof(struct globmem));

    p4_askfor_init(&(glob->MO));
    p4_barrier_init(&(glob->BA));

    printf("enter number of processes: ");
    scanf("%d",&glob->nproc);
    printf("enter the number of rows: ");
    scanf("%d",&glob->rows);
    printf("enter the number of columns: ");
    scanf("%d",&glob->columns);
    printf("enter the number of iterations: ");
    scanf("%d",&glob->goal);

    gridinit(glob->a,glob->rows,glob->columns);
    gridinit(glob->b,glob->rows,glob->columns);
    
    glob->pqbeg = glob->pqend = 0;
    for (i=1; i <= glob->rows; i++)
        queueprob(i);

    /* initialize the status vector */
    for (i=0; i < (glob->rows+2); i++)
        glob->st[i] = 0;

    printf("\nnproc\tgoal\trows\tcolumns\n");
    printf("%d \t  %d \t  %d \t  %d \n",
	   glob->nproc,glob->goal,glob->rows,glob->columns);
    for (i=1; i <= glob->nproc-1; i++) {
        p4_create(slave);
    }
    timestart = p4_clock();
    start_ustime = p4_ustimer();
    work('m');
    end_ustime = p4_ustimer();
    timeend = p4_clock();
    printf("total time %.3f seconds\n",(timeend - timestart)/1000.0);
    printf("total time %.6f seconds\n",(end_ustime-start_ustime)/1000000.0);

/* 
    printf("the resulting grid:\n");
    if (glob->goal % 2 == 0)
        printgrid(glob->a,glob->rows,glob->columns);
    else
        printgrid(glob->b,glob->rows,glob->columns);
*/
    if (glob->goal % 2 == 0)
        avg = avggrid(glob->a,glob->rows,glob->columns);
    else
        avg = avggrid(glob->b,glob->rows,glob->columns);
    printf("average value of grid = %f\n",avg);

    p4_wait_for_end(); 
}

/* "m" is the matrix, "r" is the number of rows of data (m[1]-m[r];
   m[0] and m[r+1] are boundaries), and "c" is the number of columns
   of data.
*/

gridinit(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r, c;
{
    int i, j;
    double bndavg;
    
    for (j=0; j < (c + 2); j++)
    {
        m[0][j] = phi(1,j+1);
        m[r+1][j]= phi(r+2,j+1);
    }
    for (i=1; i < (r + 2); i++)
    {
        m[i][0] = phi(i+1,1);
        m[i][c+1] = phi(i+1,c+2);
    }
    bndavg = avgbnd(m,r,c);
    printf("boundary average = %f\n",bndavg);

    /* initialize the interior of the grids to the average over the boundary*/
    for (i=1; i <= r; i++)
        for (j=1; j <= c; j++)
            /* m[i][j] = bndavg; this optimization hinders debugging */
	    m[i][j] = 0;
}

queueprob(x)
int x;
{
    glob->pq[glob->pqend] = x;
    glob->pqend = (glob->pqend + 1) % (ROWS + 1);
}

compute(p,q,r,columns)
double p[ROWS+2][COLUMNS+2];
double q[ROWS+2][COLUMNS+2];
int r;
int columns;
{
    int j;

    for (j = 1; j <= columns; j++) 
        q[r][j] = (p[r-1][j] + p[r+1][j] + p[r][j-1] + p[r][j+1]) / 4.0;
}

int putprob(r)
int r;
{
    int qprob;

    qprob = FALSE;
    glob->st[r]++;
    if (r == 1)
        glob->st[0] = glob->st[r];

    else if (r == glob->rows)
        glob->st[glob->rows+1] = glob->st[r];

    if (glob->st[r] < glob->goal)
    {
        if ((r > 1) && (glob->st[r-2] >= glob->st[r]) 
		    && (glob->st[r-1] == glob->st[r]))
	{
            queueprob(r-1);
            qprob = TRUE;
        }
        if (r < glob->rows && glob->st[r+1] == glob->st[r] 
			   && glob->st[r+1] <= glob->st[r+2])
	{
            queueprob(r+1);
            qprob = TRUE;
        }
        if (glob->st[r-1] == glob->st[r] && 
	    glob->st[r] == glob->st[r+1])
	{
            queueprob(r);
            qprob = TRUE;
        }
    }
    if (qprob)
        return(1);		/* new problem */
    else
	return(0);		/* no new problem */
}

int getprob(v)
int *v;
{
    int rc = 1;
    int *p = (int *) v;

    if (glob->pqbeg != glob->pqend) 
    {
	*p = glob->pq[glob->pqbeg];
	glob->pqbeg = (glob->pqbeg+1) % (ROWS + 1); 
	rc = 0;
    }
    return(rc);
}

P4VOID reset() 
{
}

work(who)			/* main routine for all processes */
char who;
{
    int r,rc,i;

    p4_barrier(&(glob->BA),glob->nproc);

    rc = p4_askfor(&(glob->MO),glob->nproc,getprob,(P4VOID *)&r,reset);

    while (rc == 0) {
	if ((glob->st[r] % 2) == 0)
	    compute(glob->a,glob->b,r,glob->columns);
	else 
	    compute(glob->b,glob->a,r,glob->columns);
	p4_update(&(glob->MO),putprob,(P4VOID *) r);
	/* postprob(r); */

	rc = p4_askfor(&(glob->MO),glob->nproc,getprob,(P4VOID *)&r,reset);
    }
}

printgrid(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    printf("%3d %3d %10.5f\n",i,j,m[i][j]);
}

double avggrid(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    avg += m[i][j];
    return(avg/((r+2)*(c+2)));
}

double avgbnd(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	    avg += m[i][0];
    for (i = 0; i < (r+2); i++)
	    avg += m[i][c+1];
    for (i = 1; i < (c+1); i++)
	    avg += m[0][i];
    for (i = 1; i < (c+1); i++)
	    avg += m[r+1][i];
    return(avg/(2*(c+2) + 2*(r+2) - 4)); /* average over boundary */
}

SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

LIBS = $(LIBDIR)/libp4.a $(MDEP_LIBS)

default:$(P)
	@if [ $(MACHINE) = IPSC860 -o $(MACHINE) = DELTA -o $(MACHINE) = IPSC860_SOCKETS ] ;\
	then\
		echo NOT making any monitors pgms on $(MACHINE) ;\
	else\
		make $(MFLAGS) addtwo.ask addtwo.gs grid.shmem ;\
	fi

all:$(P)
	make $(MFLAGS) addtwo.ask addtwo.gs adquad grid.shmem gridlog.shmem

grid.shmem:$(P)	grid.shmem.o $(LIBDIR)/libp4.a
	$(CC) -o grid.shmem grid.shmem.o $(LIBS)

gridlog.shmem:$(P)	gridlog.shmem.o $(LIBDIR)/libp4.a
	$(CC) -o gridlog.shmem gridlog.shmem.o $(LIBS)

addtwo.ask:$(P)	addtwo.ask.o $(LIBDIR)/libp4.a
	$(CC) -o addtwo.ask addtwo.ask.o $(LIBS) -lm

addtwo.gs:$(P)	addtwo.gs.o  $(LIBDIR)/libp4.a
	$(CC) -o addtwo.gs addtwo.gs.o $(LIBS) -lm

adquad:$(P)	adquad.o $(LIBDIR)/libp4.a
	$(CC) -o adquad adquad.o $(LIBS) -lm

realclean: clean
	$(RM) -f addtwo.ask addtwo.gs adquad grid.shmem gridlog.shmem Makefile

SHAR_EOF
if test -f 'procgroup'
then
	echo shar: over-writing existing file "'procgroup'"
fi
cat << \SHAR_EOF > 'procgroup'
local 2
SHAR_EOF
if test -f 'gridlog.shmem.c'
then
	echo shar: over-writing existing file "'gridlog.shmem.c'"
fi
cat << \SHAR_EOF > 'gridlog.shmem.c'
#define ALOG_TRACE
#include "p4.h"
#include  <sys/tmp_ctl.h>

#define ASKFOR_WORK     1
#define ASKED           2       
#define START_WORK      3
#define END_WORK        4
#define PUTTING_IN_POOL 5
#define PUT_IN_POOL     6

#define ROWS 		200
#define COLUMNS 	200
struct globmem {
    double a[ROWS+2][COLUMNS+2];
    double b[ROWS+2][COLUMNS+2];
    int st[ROWS+2], pq[ROWS+1];
    int pqbeg, pqend, goal, nproc, rows, columns;
    p4_askfor_monitor_t MO;
    p4_barrier_monitor_t BA;
} *glob;

double avggrid();
double avgbnd();

slave()
{
    ALOG_SETUP(p4_get_my_id(),ALOG_TRUNCATE);
    work('s');
    ALOG_OUTPUT;
}

phi(x,y)			/* The function on the boundary */
int x,y;
{
      return((x * x) - (y * y) + (x * y));   
}

main(argc,argv)
int argc;
char **argv;
{
    int i;
    int timestart, timeend;
    double avg;

    p4_initenv(&argc,argv);

    ALOG_ENABLE;
    ALOG_MASTER(p4_get_my_id(),ALOG_TRUNCATE);

    ALOG_DEFINE(ASKFOR_WORK,"Asking","");
    ALOG_DEFINE(ASKED,"Asked","");
    ALOG_DEFINE(START_WORK,"Working","slab %d");
    ALOG_DEFINE(END_WORK,"Finished","slab %d");
    ALOG_DEFINE(PUTTING_IN_POOL,"Putting","slab %d");
    ALOG_DEFINE(PUT_IN_POOL,"Put","slab %d");

    glob = (struct globmem *) p4_shmalloc(sizeof(struct globmem));

    p4_askfor_init(&(glob->MO));
    p4_barrier_init(&(glob->BA));

/*
    printf("enter number of processes: ");
    scanf("%d",&glob->nproc);
*/
    printf("enter the number of rows: ");
    scanf("%d",&glob->rows);
    printf("enter the number of columns: ");
    scanf("%d",&glob->columns);
    printf("enter the number of iterations: ");
    scanf("%d",&glob->goal);

    gridinit(glob->a,glob->rows,glob->columns);
    gridinit(glob->b,glob->rows,glob->columns);
    
    glob->pqbeg = glob->pqend = 0;
    for (i=1; i <= glob->rows; i++)
        queueprob(i);

    /* initialize the status vector */
    for (i=0; i < (glob->rows+2); i++)
        glob->st[i] = 0;

    p4_create_procgroup();
    glob->nproc = p4_num_total_ids();

    printf("\nnproc\tgoal\trows\tcolumns\n");
    printf("%d \t  %d \t  %d \t  %d \n",
	   glob->nproc,glob->goal,glob->rows,glob->columns);
    timestart = p4_clock();
/*
    for (i=1; i <= glob->nproc-1; i++) {
        p4_create(slave);
    }
*/
    work('m');
    timeend = p4_clock();
    printf("total time %.3f seconds\n",(timeend - timestart)/1000.0);

/* 
    printf("the resulting grid:\n");
    if (glob->goal % 2 == 0)
        printgrid(glob->a,glob->rows,glob->columns);
    else
        printgrid(glob->b,glob->rows,glob->columns);
*/
    if (glob->goal % 2 == 0)
        avg = avggrid(glob->a,glob->rows,glob->columns);
    else
        avg = avggrid(glob->b,glob->rows,glob->columns);
    printf("average value of grid = %f\n",avg);

    ALOG_OUTPUT;
    p4_wait_for_end(); 
}

/* "m" is the matrix, "r" is the number of rows of data (m[1]-m[r];
   m[0] and m[r+1] are boundaries), and "c" is the number of columns
   of data.
*/

gridinit(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r, c;
{
    int i, j;
    double bndavg;
    
    for (j=0; j < (c + 2); j++)
    {
        m[0][j] = phi(1,j+1);
        m[r+1][j]= phi(r+2,j+1);
    }
    for (i=1; i < (r + 2); i++)
    {
        m[i][0] = phi(i+1,1);
        m[i][c+1] = phi(i+1,c+2);
    }
    bndavg = avgbnd(m,r,c);
    printf("boundary average = %f\n",bndavg);

    /* initialize the interior of the grids to the average over the boundary*/
    for (i=1; i <= r; i++)
        for (j=1; j <= c; j++)
            /* m[i][j] = bndavg; this optimization hinders debugging */
	    m[i][j] = 0;
}

queueprob(x)
int x;
{
    glob->pq[glob->pqend] = x;
    glob->pqend = (glob->pqend + 1) % (ROWS + 1);
}

compute(p,q,r,columns)
double p[ROWS+2][COLUMNS+2];
double q[ROWS+2][COLUMNS+2];
int r;
int columns;
{
    int j;

    for (j = 1; j <= columns; j++) 
        q[r][j] = (p[r-1][j] + p[r+1][j] + p[r][j-1] + p[r][j+1]) / 4.0;
}

int putprob(r)
int r;
{
    int qprob;

    ALOG_LOG(p4_get_my_id(),PUTTING_IN_POOL,r,"");

    qprob = FALSE;
    glob->st[r]++;
    if (r == 1)
        glob->st[0] = glob->st[r];

    else if (r == glob->rows)
        glob->st[glob->rows+1] = glob->st[r];

    if (glob->st[r] < glob->goal)
    {
        if ((r > 1) && (glob->st[r-2] >= glob->st[r]) 
		    && (glob->st[r-1] == glob->st[r]))
	{
            queueprob(r-1);
            qprob = TRUE;
        }
        if (r < glob->rows && glob->st[r+1] == glob->st[r] 
			   && glob->st[r+1] <= glob->st[r+2])
	{
            queueprob(r+1);
            qprob = TRUE;
        }
        if (glob->st[r-1] == glob->st[r] && 
	    glob->st[r] == glob->st[r+1])
	{
            queueprob(r);
            qprob = TRUE;
        }
    }
    ALOG_LOG(p4_get_my_id(),PUT_IN_POOL,qprob,"");
    return(qprob);
}

int getprob(v)
int *v;
{
    int rc = 1;
    int *p = (int *) v;

    if (glob->pqbeg != glob->pqend) 
    {
	*p = glob->pq[glob->pqbeg];
	glob->pqbeg = (glob->pqbeg+1) % (ROWS + 1); 
	rc = 0;
    }
    return(rc);
}

P4VOID reset() 
{
}

work(who)			/* main routine for all processes */
char who;
{
    int r,rc,i;
    int my_id = p4_get_my_id();

    printf("old tmp_affinity = %d\n",tmp_affinity(my_id));
    printf("new tmp_affinity = %d\n",tmp_affinity(AFF_QUERY));

    p4_barrier(&(glob->BA),glob->nproc);

    ALOG_LOG(my_id, ASKFOR_WORK, 0, "");
    rc = p4_askfor(&(glob->MO),glob->nproc,getprob,(P4VOID *)&r,reset);
    ALOG_LOG(my_id,ASKED,rc,"");

    while (rc == 0) {
	ALOG_LOG(my_id,START_WORK,r,"");
	if ((glob->st[r] % 2) == 0)
	    compute(glob->a,glob->b,r,glob->columns);
	else 
	    compute(glob->b,glob->a,r,glob->columns);
	ALOG_LOG(my_id,END_WORK,r,"");

	p4_update(&(glob->MO),putprob,(P4VOID *) r);

	ALOG_LOG(my_id, ASKFOR_WORK, 0, "");
	rc = p4_askfor(&(glob->MO),glob->nproc,getprob,(P4VOID *)&r,reset);
	ALOG_LOG(my_id,ASKED,rc,"");
    }
}

printgrid(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    printf("%3d %3d %10.5f\n",i,j,m[i][j]);
}

double avggrid(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	for (j = 0; j < (c+2); j++)
	    avg += m[i][j];
    return(avg/((r+2)*(c+2)));
}

double avgbnd(m,r,c)
double m[ROWS+2][COLUMNS+2];
int r,c;
{
    int i,j;
    double avg = 0;

    for (i = 0; i < (r+2); i++)
	    avg += m[i][0];
    for (i = 0; i < (r+2); i++)
	    avg += m[i][c+1];
    for (i = 1; i < (c+1); i++)
	    avg += m[0][i];
    for (i = 1; i < (c+1); i++)
	    avg += m[r+1][i];
    return(avg/(2*(c+2) + 2*(r+2) - 4)); /* average over boundary */
}

SHAR_EOF
cd ..
if test ! -d 'servers'
then
	mkdir 'servers'
fi
cd 'servers'
if test -f 'README'
then
	echo shar: over-writing existing file "'README'"
fi
cat << \SHAR_EOF > 'README'
This directory contains code for two servers, a secure server and one
that is not secure.  We recommend that you use only the secure server,
we use the other one only for testing. 

This secure server can be installed for use public use, i.e. all
users can use the same copy, or you can make it available such that
individual users can start it up as they need it.  Each method has
its won advantages/disadvantages.

Secure server installation for public use
-----------------------------------------

The following steps are done as a non-root user, to install the
server.  We assume that it has been made as part of the top-level
p4 installation.


1. Decide in which directory the server will live. By default it
goes into /etc. There will be two files in that directory: serv_p4
(the server), and serv_p4.log (a logfile created by serv_p4). This
will be the SERVER_HOME directory.

2. Edit Makefile to reflect the correct value for SERVER_HOME.

The values for P4_PORT and P4_PORT_NAME should not need to be changed.

3. Do a "make all". 

As root:

4. Add an entry to /etc/services. The contents of services.entry
should work.

If you are on a NIS/YellowPages network, this step need only be done
on the NIS server machine. After making the change, cd to the NIS
directory (usually /var/yp) and do a "make".

5. Add an entry to /etc/inetd.conf for the server. See the manual page
for inetd(8) for the correct syntax. One of the examples in
inetd.entry is probably correct for your system. 

Ensure that the server will be run as root.

Ensure that the pathname in /etc/inetd.conf points to the correct place.

6. Make inetd aware of the new entry. On newer Unices, you should be
able to do a "kill -HUP <inetd process id>". If that does not work,
kill ("kill <inetd process id") and restart ("/etc/inetd &") inetd. If
all else fails, reboot the machine.

7. Do a "make install". This will copy serv_p4 to
the SERVER_HOME directory.

--------------------------------------------------

The P4 secure server should now be available. If you have problems,
first try to telnet directly to the server:
	
	% telnet <machine> serv_p4

If you get the error 

	serv_p4: bad port number

the /etc/services entry is incorrect. Did you do a make in /var/yp on
a NIS system?

If you get the error

	telnet: connect: Connection refused

inetd probably didn't get restarted correctly, or there is a syntax
error in the /etc/inetd.conf file. 




Installing the server for private use
-------------------------------------

Merely make the server in a directory where all users can execute it,
and then note the following:


The server has several options (I didn't explain these earlier).

    o   -d says to run in "daemon" mode. This is as opposed to
        "inetd" mode: it will loop accepting connections itself,
        instead of expecting fd 0 to be the net connection.

    o   "-p port" says to listen on port, instead of the default port.
        If port is set to 0, the server will pick a port and print
        its value to stdout.

    o   "-l logfile" sets the logfile.

    o   "-D" says run in debug mode. By default, in daemon mode the
        server will fork and disconnect itself from the terminal. The
       -D flag disables this.

If a non-root user runs the server, it will only run processes for that
user. It will also assume "-p 0" and "-D" and set the default logfile
to "P4Server.log.<pid>". 




Using P4 with the server
------------------------

In order for the server to execute a slave process, several
requirements must be met. 

    o	The name of the executable must be supplied as a full pathname
        in the procgroup file

    o	A file called .p4apps must exist in the home directory of the
	user under whose account the process will be run. The .p4apps
	file contains a list of filenames for executable p4 slaves,
	one per line. An executable must be listed in this file for
	the server to execute it.

	For example, if the procgroup file entry is

		remote 1 /usr/user/sr_slave ruser
	
	the file ~ruser/.p4apps must exist on the machine called
	"remote" and contain the line

		/usr/user/sr_slave

        In the file .p4apps blank lines and lines starting with '#'
        are ignored.


The server will require the user to supply a password if the proper
/etc/hosts.equiv or ~user/.rhosts files do not have the appropriate
entries. In other words, if rlogin requires a password, so will the P4
server.
SHAR_EOF
if test -f 'inetd.entry.base'
then
	echo shar: over-writing existing file "'inetd.entry.base'"
fi
cat << \SHAR_EOF > 'inetd.entry.base'

Entry for older versions of Unix (e.g. Ultrix 4.x):

P4_PORT_NAME	stream	tcp	nowait	SERVER_HOME/serv_p4 serv_p4

Entry for newer BSD4.3 based Unices (e.g. SunOS, Sequent Dynix, IRIX, 
Alliant Concentrix, NeXT):

P4_PORT_NAME stream  tcp      nowait  root SERVER_HOME/serv_p4 serv_p4
SHAR_EOF
if test -f 'iserv_p4.c'
then
	echo shar: over-writing existing file "'iserv_p4.c'"
fi
cat << \SHAR_EOF > 'iserv_p4.c'
#include "p4.h"
#include "p4_sys.h"

extern int errno;
extern char *sys_errlist[];

VOID reaper();

main(argc,argv)
int argc;
char **argv;
{
    int listen_fd, connection_fd;
    int port;
    int done;
    struct sockaddr_in from;
    int fromlen;
    
    p4_initenv(&argc,argv);
    sprintf(whoami, "server"); /* alter p4 environment */

    net_setup_listener(5, UNRESERVED_PORT, &listen_fd);

#ifdef P4BSD
    signal(SIGCHLD, reaper);
#endif
#ifdef P4SYSV
    signal(SIGCLD, reaper);
#endif

    done = FALSE;
    while (!done)
    {
	fromlen = sizeof(from);
	connection_fd = accept(listen_fd, &from, &fromlen);
	if (connection_fd == -1)
	{
	    if (errno == EINTR)
		continue;
	    else
	    {
		perror("server accept");
		exit(1);
	    }
	}
	p4_dprintfl(20,"accepted on %d\n", connection_fd);
	done = process_connection(connection_fd);
	shutdown(connection_fd, 2);
	close(connection_fd);
    }

    shutdown(listen_fd, 2);
    close(listen_fd);
}

process_connection(fd)
int fd;
{
struct net_message_t msg;
int type;
char *pgm;
char *host;
char *am_slave;
int port;

    if (net_recv(fd, &msg, sizeof(msg)) == NET_RECV_EOF)
	return(FALSE);

    type = p4_n_to_i(msg.type);

    switch (type)
    {
	case NET_EXEC:
	    pgm = msg.pgm;
	    host = msg.host;
	    am_slave = msg.am_slave;
	    port = p4_n_to_i(msg.port);
	    p4_dprintfl(20,"server got exec msg: pgm=%s host=%s port=%d am_slave=%s\n",
		   pgm, host, port, am_slave);
	    exec_pgm(host, pgm, port, am_slave);
	    msg.type = p4_i_to_n(NET_RESPONSE);
	    msg.success = p4_i_to_n(1);
	    net_send(fd, &msg, sizeof(msg), FALSE);
	    return(FALSE);

	case NET_DONE:
	    p4_dprintfl(20,"server got done message\n");
	    return(TRUE);

	default:
	    p4_dprintfl(20,"server got unknown message type %d\n", type);
	    return(FALSE);
    }
}
	
VOID exec_pgm(host, pgm, port, am_slave)
char *host, *pgm, *am_slave;
int port;
{
int pid;
int rc;
char shortpgm[100];
char sport[10];
char *s;

    if ((s = rindex(pgm, '/')) != NULL)
	strcpy(shortpgm, s + 1);
    else
	strcpy(shortpgm, pgm);

    sprintf(sport, "%d", port);

    p4_dprintfl(20,"exec_pgm: pgm=%s short=%s sport=%s\n",
		    pgm, shortpgm, sport);

    fflush(stdout);
    pid = fork();  /* Not fork_p4 here as don't want interrupts on error */
    if (pid < 0)
	p4_error("exec_pgm fork",pid);
    if (pid == 0)
    {
	/* The child */

	/****
	  RB This line is typically used, but some versions
	  of exec seem to have a bug requiring the full pgm
	  name to be in both arg positions.

	rc = execl(pgm, shortpgm, host, sport, am_slave, NULL);
	****/

	rc = execl(pgm, pgm, host, sport, am_slave, NULL);

	if (rc < 0)
	    p4_error("exec_pgm execl",rc);
	exit(1);
    }
}

VOID reaper()
{
int status;

int pid;
    
    p4_dprintfl(20,"server: entering reaper\n");
    pid = wait(&status);
    p4_dprintfl(20,"server: pid %d died with status %d\n", pid, status);
}

int slave() /* dummy proc */
{
    return(0);
}
SHAR_EOF
if test -f 'killiserv_p4.c'
then
	echo shar: over-writing existing file "'killiserv_p4.c'"
fi
cat << \SHAR_EOF > 'killiserv_p4.c'
#include "p4.h"
#include "p4_sys.h"

int debug_level;
char whoami[100];
char local_domain[100];

main(argc,argv)
int argc;
char **argv;
{
    char host[100];
    int i;
    
    *local_domain = 0;
    debug_level = 0;
    sprintf(whoami, "kill_server_%d", getpid());
    
    if (argc == 1)
    {
	get_qualified_hostname(host);
	kill_server(host);
    }
    else
    {
	for (i = 1; i < argc; i++)
	{
	    kill_server(argv[i]);
	}
    }
}

VOID kill_server(host)
char *host;
{
    struct net_message_t msg;
    int fd;
    
    fd = net_conn_to_listener(host, UNRESERVED_PORT);
    if (fd == -1)
    {
	printf("couldn't connect to server on %s\n", host);
	return;
    }
    printf("killing server on %s\n", host);
    
    msg.type = p4_i_to_n(NET_DONE);
    
    net_send(fd, &msg, sizeof(msg), FALSE);
    close(fd);
}

int slave() /* dummy proc */
{
    return(0);
}

SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
SERVER_HOME = /etc

P4_PORT = 753
P4_PORT_NAME = serv_p4

CHMOD = chmod
CHOWN = chown
INSTALL = /bin/cp -i
SED   = sed
LIBLINK = ../bin

P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib

default:
	@if [ $(MACHINE) = SUN -o $(MACHINE) = SYMMETRY \
                               -o $(MACHINE) = DEC5000 ] ;\
	then\
		make $(MFLAGS) serv_p4 example_configs ;\
	else\
		echo not making server on $(MACHINE) since not tested there ;\
	fi

all: serv_p4 example_configs iserv_p4 killiserv_p4

example_configs: inetd.entry services.entry

serv_p4: serv_p4.o
	$(CC) $(CFLAGS) -o $(LIBLINK)/serv_p4 serv_p4.o

iserv_p4:$(P)  iserv_p4.o
	$(CC) $(CFLAGS) -o $(LIBLINK)/iserv_p4 iserv_p4.o $(LIBDIR)/libp4.a

killiserv_p4:$(P)  killiserv_p4.o
	$(CC) $(CFLAGS) -o $(LIBLINK)/killiserv_p4 killiserv_p4.o $(LIBDIR)/libp4.a

inetd.entry: inetd.entry.base Makefile
	$(SED) -e 's,P4_PORT_NAME,$(P4_PORT_NAME),g' \
	    -e 's,P4_PORT,$(P4_PORT),g' \
	    -e 's,SERVER_HOME,$(SERVER_HOME),g' \
	    inetd.entry.base > inetd.entry

services.entry: services.entry.base Makefile
	$(SED) -e 's,P4_PORT_NAME,$(P4_PORT_NAME),g' \
	    -e 's,P4_PORT,$(P4_PORT),g' \
	    -e 's,SERVER_HOME,$(SERVER_HOME),g' \
	    services.entry.base > services.entry

install: serv_p4
	$(INSTALL) serv_p4 $(SERVER_HOME)
	$(CHOWN) root $(SERVER_HOME)/serv_p4
	$(CHMOD) 500 $(SERVER_HOME)/serv_p4

realclean:	clean
	$(RM)  -f *~ \#* inetd.entry services.entry \
		     serv_p4 Makefile P4Server.Log.* \
		     iserv_p4 killiserv_p4
SHAR_EOF
if test -f 'serv_p4.c'
then
	echo shar: over-writing existing file "'serv_p4.c'"
fi
cat << \SHAR_EOF > 'serv_p4.c'
#include "p4.h"
#include "p4_sys.h"

/****
#include <stdio.h>
#include <ctype.h>
#include <pwd.h>
#include <errno.h>
#include <signal.h>
#include <netdb.h>
#include <sys/socket.h>
****/

#include <sys/types.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <sys/ioctl.h>
#include <time.h>


#ifdef P4BSD
#include <strings.h>
#endif

#ifdef P4SYSV
#include <string.h>
#endif

#if defined(SYMMETRY)
#define NEED_GETOPT
#endif

#ifdef NEED_GETOPT

/* This is from the released BSD sources lib/libc/getopt.c */

/*
 * get option letter from argument vector
 */
int	opterr = 1,		/* if error message should be printed */
	optind = 1,		/* index into parent argv vector */
	optopt;			/* character checked for validity */
char	*optarg;		/* argument associated with option */

#else

extern char *optarg;

#endif

#define MAXARGS 256

#ifndef LOGFILE
#define LOGFILE "/etc/serv_p4.log"
#endif

#define notice2(a,b) {sprintf(tmpbuf, a, b); notice(tmpbuf);}
#define notice3(a,b,c) {sprintf(tmpbuf, a, b,c); notice(tmpbuf);}
#define failure2(a,b) {sprintf(tmpbuf, a, b); failure(tmpbuf);}

extern char *crypt();

extern char *sys_errlist[];
extern int errno;

char tmpbuf[1024];
char *fromhost;

char logfile[1024];
FILE *logfile_fp;

#define DEFAULT_PORT 753

int daemon_mode;
int daemon_port;
int debug;

char *this_username;
int this_uid;

void doit();
void execute();
int getline();
void failure();
void notice();
int net_accept();
void net_setup_listener();
void net_setup_anon_listener();
void error_check();
char *timestamp();

char *save_string();

void reaper()
{
    int i;
    wait(&i);
}

main(argc, argv)
int argc;
char **argv;
{
    int c;
    struct sockaddr_in name;
    int namelen;

    if (getuid() == 0)
    {
	strcpy(logfile, LOGFILE);
	daemon_port = DEFAULT_PORT;
    }
    else
    {
	sprintf(logfile, "P4Server.Log.%d", getpid());
	daemon_port = 0;
	debug = 1;
    }

    namelen = sizeof(name);
    if (getpeername(0, (struct sockaddr *) &name, &namelen) < 0)
	daemon_mode = 1;
    else
	daemon_mode = 0;
    
    while ((c = getopt(argc, argv, "Ddp:l:")) != EOF)
    {
	switch (c)
	{
	case 'D':
	    debug++;
	    break;
	    
	case 'd':
	    daemon_mode++;
	    break;

	case 'p':
	    daemon_port = atoi(optarg);
	    break;

	case 'l':
	    strcpy(logfile, optarg);
	    break;

	case '?':
	default:
	    fprintf(stderr, "Usage: %s [-d] [-D] [-p port] [-l logfile]\n",argv[0]);
	    exit(1);
	}
    }

    if ((logfile_fp = fopen(logfile, "a")) == NULL)
    {
	if (getuid() != 0)
	{
	    printf("Cannot open logfile, disabling logging\n");
	    logfile_fp = fopen("/dev/null", "w");
	}
	else
	{
	    fprintf(stderr, "Cannot open logfile %s: %s\n",
		    logfile, sys_errlist[errno]);
	    exit(1);
	}
    }
    else
	printf("Logging to %s\n", logfile);

    setbuf(logfile_fp, NULL);

    fprintf(logfile_fp, "%s pid=%d starting at %s",
	    argv[0], getpid(), timestamp());

    if (daemon_mode)
    {
	int port, lfd, fd, pid;

	signal(SIGCHLD, reaper);

	if (daemon_port == 0)
	{
	    net_setup_anon_listener(2, &daemon_port, &lfd);
	}
	else
	{
	    net_setup_listener(2, daemon_port, &lfd);
	}

	if (debug || daemon_port != DEFAULT_PORT)
	    printf("Listening on %d\n", daemon_port);
	    
	if (!debug)
	{
	    if (fork())
		exit();

	    for (fd = 0; fd < 10; fd++)
		if (fd != lfd && fd != fileno(logfile_fp))
		    close(fd);
	    
#ifdef P4SYSV
	    fd = open ("/dev/console", O_RDWR);
	    if (fd < 0)
		fd = open ("/dev/tty", O_RDWR);
	    if (fd < 0)
		fd = open ("/dev/null", O_RDWR);
	    (void) dup2(STDIN, STDOUT);
	    (void) dup2(STDIN, STDERR);
	    (void) setpgrp();
#else
	    (void) open("/", 0);
	    (void) dup2(0, 1);
	    (void) dup2(0, 2);
	    fd = open("/dev/tty", O_RDWR);
	    if (fd >= 0) {
		ioctl(fd, TIOCNOTTY, 0);
		(void) close(fd);
	    }
#endif
	}

	while (1)
	{
	    fd = net_accept(lfd);

	    pid = fork();

	    if (pid < 0)
	    {
		fprintf(logfile_fp, "Fork failed: %s\n",
			sys_errlist[errno]);
		exit(pid);
	    }
	    if (pid == 0)
	    {
		int ttyfd;

		ttyfd = open("/dev/tty",O_RDWR);
		if (ttyfd >= 0)
		{
		    ioctl(ttyfd, TIOCNOTTY, 0);
		    close(ttyfd);
		}

		close(0);
		close(1);
		close(2);
		close(lfd);
		
		dup2(fd, 0);
		dup2(fd, 1);
		dup2(fileno(logfile_fp), 2);

		doit(0);
		exit(0);
	    }
	    close(fd);
	}
    }
    else
    {
	doit(0);
    }
	
}

void doit(fd)
int fd;
{
    struct sockaddr_in name;
    int namelen;
    struct hostent *hp;

    struct passwd *pw;
    char client_user[80], server_user[80];
    char pgm[1024], pgm_args[1024];
    char *user_home;
    int superuser;
    int valid;
    FILE *fp;
    int stdout_port;
    char stdout_port_str[16];

    char filename[1024], progline[1024];
    struct stat statbuf;

    this_uid = getuid();
    pw = getpwuid(this_uid);
    if (pw == NULL)
    {
	fprintf(logfile_fp, "Cannot get pw entry for user %d\n", this_uid);
	exit(1);
    }
    this_username = save_string(pw->pw_name);

    if (this_uid != 0)
	fprintf(logfile_fp, "WARNING: Not run as root\n");

    setbuf(stdout, NULL);

    fprintf(logfile_fp, "Got connection at %s", timestamp());

    namelen = sizeof(name);

    if (getpeername(fd, (struct sockaddr *) &name, &namelen) != 0)
    {
	fprintf(logfile_fp, "getpeername failed: %s\n",
		sys_errlist[errno]);
	exit(1);
    }

    fromhost = inet_ntoa(name.sin_addr);
    
    hp = gethostbyaddr((char *) &name.sin_addr,
		       sizeof(name.sin_addr),
		       (int) name.sin_family);
    if (hp == NULL)
	failure2("Cannot get remote address for %s", fromhost);

    fromhost = hp->h_name;

    if (!getline(client_user, sizeof(client_user)))
	failure("No client user");

    if (!getline(server_user, sizeof(server_user)))
	failure("No server user");

    pw = getpwnam(server_user);
    if (pw == NULL)
	failure2("No such user: %s\n", server_user);

    if (this_uid != 0 && this_uid != pw->pw_uid)
	failure2("Server is not running as root. Only %s can start processes\n",
		 this_username);

    user_home = pw->pw_dir;
    superuser = (pw->pw_uid == 0);

    valid = ruserok(fromhost, superuser, client_user, server_user);

    if (valid != 0)
    {
	char user_pw[80];
	char *xpw;
	
	printf("Password\n");
	if (!getline(user_pw, sizeof(user_pw)))
	    failure("No server user");

	xpw = crypt(user_pw, pw->pw_passwd);
	if (strcmp(pw->pw_passwd, xpw) != 0)
	    failure("Invalid password");

	printf("Proceed\n");
    }
    else
	printf("Proceed\n");

    sprintf(tmpbuf, "authenticated client_id=%s server_id=%s\n",
	    client_user, server_user);
    notice(tmpbuf);

    if (!getline(pgm, sizeof(pgm)))
	failure("No pgm");

    if (!getline(pgm_args, sizeof(pgm_args)))
	failure("No pgm args");

    notice2("got args %s", pgm_args);

    if (pgm[0] != '/')
	failure2("%s is not a full pathname", pgm);

    if (this_uid == 0)
    {
	if (seteuid(pw->pw_uid) != 0)
	    failure2("seteuid failed: %s", sys_errlist[errno]);
    }
    
    sprintf(filename, "%s/.p4apps", user_home);
    valid = 0;
    if ((fp = fopen(filename,"r")) != NULL)
    {
	char *s1, *s2;

	if (fstat(fileno(fp), &statbuf) != 0)
	    failure2("cannot stat %s", filename);

	if (statbuf.st_mode & 077)
	    failure(".p4apps readable by others");

	while (fgets(progline, sizeof(progline), fp) != NULL)
	{
	    s1 = progline;
	    while (*s1 && isspace(*s1))
		s1++;
	    if (*s1 == '\0' || *s1 == '#')
		continue;

	    s2 = s1;
	    while (*s2 && !isspace(*s2))
		s2++;
	    *s2 = 0;
	    if (strcmp(pgm, s1) == 0)
	    {
		valid = 1;
		break;
	    }
	}
	fclose(fp);
    }
    if (!valid)
	failure2("Invalid program %s", pgm);
    
    if (stat(pgm, &statbuf) != 0)
	failure2("Cannot stat %s", pgm);

    if (!(statbuf.st_mode & 0111))
	failure2("Cannot execute %s", pgm);

    notice3("executing %s %s", pgm, pgm_args);

    /*********/
    if (!getline(stdout_port_str, sizeof(stdout_port_str)))
	failure("No stdout");
    else
	stdout_port = atoi(stdout_port_str);

    notice2("got stdout_port %d", stdout_port);
    /*********/

    execute(pgm, pgm_args, pw->pw_uid, stdout_port, hp);
	    
}

void execute(pgm, pgm_args, uid, stdout_port, hp)
char *pgm, *pgm_args;
int uid, stdout_port;
struct hostent *hp;
{
    int p[2];
    int rd, wr;
    int pid, n;
    char *args[MAXARGS];
    int nargs;
    char *s, *end;
    int i;
    char buf[1024];
    int stdout_fd;
    char tempbuf[100];

    s = pgm_args;
    while (*s && isspace(*s))
	s++;

    args[0] = pgm;

    nargs = 1;
    while (*s)
    {
	args[nargs] = s;

	while (*s && !isspace(*s))
	    s++;

	end = s;

	while (*s && isspace(*s))
	    s++;

	*end = 0;
	nargs++;
	if (nargs + 1>= MAXARGS)
	    failure("Too many arguments to pgm");
    }

    args[nargs] = NULL;

    if (pipe(p) != 0)
	failure2("Cannot create pipe: %s", sys_errlist[errno]);

    rd = p[0];
    wr = p[1];

    if (fcntl(wr, F_SETFD, 1) != 0)
	failure2("fcntl F_SETFD failed: %s", sys_errlist[errno]);

    if (this_uid == 0)
    {
	if (seteuid(0) != 0)
	    failure2("cannot seteuid: %s", sys_errlist[errno]);
	
	if (setreuid(uid, uid) != 0)
	    failure2("cannot setreuid: %s", sys_errlist[errno]);
    }
    
    pid = fork();
    if (pid < 0)
	failure2("fork failed: %s", sys_errlist[errno]);

    if (pid == 0)
    {
	close(rd);

	close(0);
	open("/dev/null", O_RDONLY);

	stdout_fd = connect_to_listener(hp,stdout_port);
	notice2("stdout_fd=%d", stdout_fd);
	close(1);
	dup(stdout_fd);
	/* open("/dev/null", O_WRONLY); */

	close(2);
	open("/dev/null", O_WRONLY);

	/*****
	strcpy(tempbuf,"writing this to stdout_fd");
	write(stdout_fd,tempbuf,strlen(tempbuf)+1);
	strcpy(tempbuf,"writing this to real stdout");
	write(stdout,tempbuf,strlen(tempbuf)+1);
	*****/

	if (execv(pgm, args) != 0)
	{
	    sprintf(tmpbuf, "Exec failed: %s\n", sys_errlist[errno]);
	    write(wr, tmpbuf, strlen(tmpbuf));
	    exit(0);
	}
    }

    close(wr);

    if ((n = read(rd, buf, sizeof(buf))) > 0)
    {
	buf[n] = 0;
	s = index(buf, '\n');
	if (s)
	    *s = 0;
	
	failure2("child failed: %s", buf);
    }
    printf("Success: Child %d started\n", pid);
    notice2("Child %d started", pid);
}

int getline(str, len)
char *str;
int len;
{
    char *s;
    
    if (fgets(str,  len, stdin) == NULL)
	return 0;

    if ((s = index(str, '\n')) != NULL)
	*s = 0;
    if ((s = index(str, '\r')) != NULL)
	*s = 0;
    return 1;
}
    

void failure(s)
char *s;
{
    printf("Failure <%s>: %s\n", fromhost, s);
    fprintf(logfile_fp, "Failure <%s>: %s\n", fromhost, s);
    fflush(logfile_fp);
    exit(1);
}

void notice(s)
char *s;
{
    fprintf(logfile_fp, "Notice <%s>: %s\n", fromhost, s);
    fflush(logfile_fp);
}


/*
  Accept a connection on socket skt and return fd of new connection.
 */
int net_accept(skt)
int skt;
{
struct sockaddr_in from;
int fromlen;
int skt2;
int gotit;

    fromlen = sizeof(from);
    gotit = 0;
    while (!gotit)
    {
	skt2 = accept(skt, (struct sockaddr *) &from, &fromlen);
	if (skt2 == -1)
	{
	    if (errno == EINTR)
		continue;
	    else
		error_check(skt2, "net_accept accept");
	}
	else
	    gotit = 1;
    }

    return(skt2);
}

void net_setup_listener(backlog, port, skt)
int backlog, port, *skt;
{
int sinlen;
struct sockaddr_in sin, from;

    *skt = socket(AF_INET, SOCK_STREAM, 0);

    error_check(*skt,"net_setup_anon_listener socket");

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = INADDR_ANY;
    sin.sin_port = htons(port);

    sinlen = sizeof(sin);

    error_check(bind(*skt,(struct sockaddr *) &sin,sizeof(sin)),
		   "net_setup_listener bind");


    error_check(listen(*skt, backlog), "net_setup_listener listen");
}

void net_setup_anon_listener(backlog, port, skt)
int backlog, *port, *skt;
{
int sinlen;
struct sockaddr_in sin, from;

    *skt = socket(AF_INET, SOCK_STREAM, 0);

    error_check(*skt,"net_setup_anon_listener socket");

    sin.sin_family = AF_INET;
    sin.sin_addr.s_addr = INADDR_ANY;
    sin.sin_port = htons(0);

    sinlen = sizeof(sin);

    error_check(bind(*skt,(struct sockaddr *) &sin,sizeof(sin)),
		   "net_setup_anon_listener bind");


    error_check(listen(*skt, backlog), "net_setup_anon_listener listen");

    getsockname(*skt, (struct sockaddr *) &sin, &sinlen);
    *port = ntohs(sin.sin_port);
}

void error_check(val, str)
int val;
char *str;
{
    if (val < 0)
    {
	fprintf(logfile_fp, "%s: %s\n",
		str,
		sys_errlist[errno]);
	exit(1);
    }
}


char *timestamp()
{
    long clock;
    struct tm *tmp;

    clock = time(0L);
    tmp = localtime(&clock);
    return asctime(tmp);
}

char *save_string(s)
char *s;
{
    char *rc = (char *) malloc(strlen(s) + 1);
    strcpy(rc, s);
    return rc;
}

#ifdef NEED_GETOPT
/* This is from the released BSD sources lib/libc/getopt.c */
/*
 * Copyright (c) 1987 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#define	BADCH	(int)'?'
#define	EMSG	""

int
getopt(nargc, nargv, ostr)
	int nargc;
	char **nargv;
	char *ostr;
{
	static char *place = EMSG;		/* option letter processing */
	register char *oli;			/* option letter list index */
	char *p;

	if (!*place) {				/* update scanning pointer */
		if (optind >= nargc || *(place = nargv[optind]) != '-') {
			place = EMSG;
			return(EOF);
		}
		if (place[1] && *++place == '-') {	/* found "--" */
			++optind;
			place = EMSG;
			return(EOF);
		}
	}					/* option letter okay? */
	if ((optopt = (int)*place++) == (int)':' ||
	    !(oli = index(ostr, optopt))) {
		/*
		 * if the user didn't specify '-' as an option,
		 * assume it means EOF.
		 */
		if (optopt == (int)'-')
			return(EOF);
		if (!*place)
			++optind;
		if (opterr) {
			if (!(p = rindex(*nargv, '/')))
				p = *nargv;
			else
				++p;
			(void)fprintf(stderr, "%s: illegal option -- %c\n",
			    p, optopt);
		}
		return(BADCH);
	}
	if (*++oli != ':') {			/* don't need argument */
		optarg = NULL;
		if (!*place)
			++optind;
	}
	else {					/* need an argument */
		if (*place)			/* no white space */
			optarg = place;
		else if (nargc <= ++optind) {	/* no arg */
			place = EMSG;
			if (!(p = rindex(*nargv, '/')))
				p = *nargv;
			else
				++p;
			if (opterr)
				(void)fprintf(stderr,
				    "%s: option requires an argument -- %c\n",
				    p, optopt);
			return(BADCH);
		}
	 	else				/* white space */
			optarg = nargv[optind];
		place = EMSG;
		++optind;
	}
	return(optopt);				/* dump back option letter */
}

#endif

static int connect_to_listener(hp,stdout_port)
struct hostent *hp;
int stdout_port;
{
    int conn;
    int rc;
    struct sockaddr_in addr;

    conn = socket(AF_INET, SOCK_STREAM, 0);
    if (conn < 0)
    {
	failure("connect_to_listener: socket failed");
    }

    addr.sin_family = hp->h_addrtype;
    addr.sin_port = htons(stdout_port);
    bcopy(hp->h_addr, &addr.sin_addr, hp->h_length);

    rc = connect(conn, (struct sockaddr *) & addr, sizeof(addr));
    if (rc < 0)
    {
	failure("connect_to_listener: connect failed");
    }

    return conn;
}

SHAR_EOF
if test -f 'services.entry.base'
then
	echo shar: over-writing existing file "'services.entry.base'"
fi
cat << \SHAR_EOF > 'services.entry.base'
P4_PORT_NAME         P4_PORT/tcp         # P4 secure server
SHAR_EOF
cd ..
if test ! -d 'usc'
then
	mkdir 'usc'
fi
cd 'usc'
if test -f 'usc.h'
then
	echo shar: over-writing existing file "'usc.h'"
fi
cat << \SHAR_EOF > 'usc.h'
/*
 * USC.H  (Public header file for the Microsecond Clock package)
 *     This header file has to be included by an application using
 * the USC function calls.
 *
 * Written by:  Arun Nanda    (07/17/91)
 * Modified by R. Butler
 *
 * The following machine-specific libraries need to be linked in
 * with the application using the UST functions:
 *
 */

#ifndef _USC_DEFS_   /* takes care of multiple inclusion of this file */
#define _USC_DEFS_


#ifdef SUN_IPC
#define SUN
#endif

#ifdef HP_IPC
#define HP
#endif

#ifdef RS6000_IPC
#define RS6000
#endif

#ifdef IBM3090_IPC
#define IBM3090
#endif

#ifdef TITAN_IPC
#define TITAN
#endif

#ifdef SGI_IPC
#define SGI
#endif



/* ---------------------
 Global declarations
--------------------- */

typedef unsigned long usc_time_t;

#ifndef VOID
#    if defined(BALANCE) || defined(FX8)
#        define VOID int
#    else
#        define VOID void
#    endif
#endif

/* --------------------------------
 Machine dependent declarations
-------------------------------- */

#if defined(MULTIMAX)

     extern unsigned *usc_multimax_timer;

#endif


#if defined(SYMMETRY) || defined(SYMMETRY_PTX)

#ifndef GETUSCLK
#    include <usclkc.h>
#endif
#endif 

extern usc_time_t usc_MD_rollover_val;

/* -----------------------
 user interface macros
----------------------- */

#if defined(MULTIMAX)

#    define usc_clock() ((usc_time_t) *usc_multimax_timer)
#    define usc_rollover_val()  (usc_MD_rollover_val)

#else

#if defined(SYMMETRY) || defined(SYMMETRY_PTX)

#    define usc_clock() ((usc_time_t) getusclk())
#    define usc_rollover_val()  (usc_MD_rollover_val)

#else

#if defined(TC_2000) || defined(TC_2000_TCMP)

#    define usc_clock() usc_MD_clock()
#    define usc_rollover_val()  (usc_MD_rollover_val)

#else

#if defined (IPSC860)

#    define usc_clock() usc_MD_clock()
#    define usc_rollover_val()  (usc_MD_rollover_val)

#else

#if defined(FX2800)  ||   defined(FX2800_SWITCH)
#    define usc_clock() usc_MD_clock()
#    define usc_rollover_val()  (usc_MD_rollover_val)
#else 

#if defined(SUN) || defined(DEC5000) || defined(HP) \
    || defined(IBM3090) || defined(RS6000) \
    || defined(NEXT) || defined(TITAN) || defined(GP_1000) \
    || defined(KSR) \
    || defined(SGI) || defined(FX8)

#    define usc_clock() usc_MD_clock()
#    define usc_rollover_val()  (usc_MD_rollover_val * 1000000 - 1)

#else

#    define usc_clock() 0
#    define usc_rollover_val() 0

#endif
#endif
#endif
#endif
#endif
#endif

/* ----------------------
  function prototypes
---------------------- */

VOID usc_init();
usc_time_t usc_MD_clock();

#endif
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ..
INCLUDEDIR = $(P4_HOME_DIR)/include
LIBDIR = $(P4_HOME_DIR)/lib
LIBS = $(MDEP_LIBS)
LIBLINK = ../bin

USC_HDR = usc_sys.h usc.h
USC_SRC = usc.c
USC_OBJ = usc.o

default:	
	make libusc.a

all:	
	make libusc.a

CFLAGS = -D$(MACHINE)

usc.o:$(P)
	$(CC) $(CFLAGS) -I$(INCLUDEDIR) -c usc.c 

$(USC_OBJ):  $(USC_SRC) $(USC_HDR)


libusc.a:  $(USC_OBJ)
	$(AR) $(LIBDIR)/libp4.a $?
	$(RANLIB) $(LIBDIR)/libp4.a
#	$(AR) $@ $?
#	$(RANLIB) $@

realclean:	clean
	$(RM) -f libusc.a Makefile
	@cd usctest; make clean

SHAR_EOF
if test -f 'README_USC'
then
	echo shar: over-writing existing file "'README_USC'"
fi
cat << \SHAR_EOF > 'README_USC'
 --------------------------------------------------------------------
 | README (Summary documentation for the Microsecond Clock package) |
 |                                                                  |
 | This file briefly describes the functionality  and the user      |
 | interface for the USC system.                                    |
 |                                                                  |
 | Written by:  Arun Nanda    (07/17/91)                            |
 |                                                                  |
 --------------------------------------------------------------------

This directory contains the source files for the Microsecond Clock
package (USC).

The machine classes that are supported are those that have a hardware
clock available with at least 1 microsecond resolution.  In that case,
true microsecond resolution is obtained.  For other machines (like SUN),
any real-time clock that is available (e.g. gettimeofday) is used and
scaled to a microsecond value.  Thus, the clock value returned on such 
machines will not yield true microsecond resolution.


CLOCK DATA TYPE
===============

For the purpose of maintaining portability of an application across a
range of architectures, a machine independent data type, denoted by
"usc_time_t", has been defined in the USC package.  The current clock
value and the clock rollover point (see below) are returned as data type
usc_time_t.  Hence, all variables within an user application used to
store and manipulate microsecond clock values must be declared as type
"usc_time_t".


MICROSECOND CLOCK INTERFACE
===========================

The package comprises of the following macros/functions that are
provided via the include file "usc.h" and the object module "usc.o".

	usc_init()        :  (with no parameters)
			     This function must be called exactly once
			     at the beginning before the microsecond
			     clock can be used.

	usc_clock()       :  (with no parameters)
			     This function returns the current value of
			     the microsecond clock as a data item of
			     type "usc_time_t".  The microsecond clock
			     is a free running counter that rolls over
			     after the maximum value which is machine
			     dependent.  No assumptions should be made
			     about what the rollover point for a given
			     machine is.  Instead, the generic macro
			     usc_rollover_val() should used to determine
			     this value.

	usc_rollover_val():  (with no parameters)
			     This macro is a generic interface to obtain
			     the maximum value that the microsecond
			     clock can take before rolling over to begin
			     a new cycle.  This macro must always be used 
			     to determine the clock rollover point.

An application that uses the USC functional interface must do the
following:

 1) Include the file "usc.h" at the beginning of the program.
 2) Link the application with the object module "usc.o" and the machine-
    specific library mentioned below.


INCLUDE FILE
============

	usc.h


OBJECT FILE TO LINK
===================

	usc.o


ADDITIONAL MACHINE DEPENDENT LIBRARIES
======================================

The application will also have to be linked with the following machine
dependent libraries in order to enable the microsecond clock to function
properly:

                 MULTIMAX        -lpp
                 BALANCE         -lseq
                 SYMMETRY        -lseq


LIST OF MACHINES SUPPORTED
==========================

		ALLIANT
		BALANCE 
		BFLY1 
		BFLY2 
		BFLY2_TCMP 
		DELTA 
		IBM3090
		RS6000 
		IPSC860_HOST 
		IPSC860_NODE 
		MULTIMAX 
		NEXT 
		SGI 
		SUN 
		SYMMETRY 
		TITAN 
SHAR_EOF
if test -f 'usc.c'
then
	echo shar: over-writing existing file "'usc.c'"
fi
cat << \SHAR_EOF > 'usc.c'
/*
 * USC.C  (Source file for the Microsecond Clock package)
 *
 * Written by:  Arun Nanda    (07/17/91)
 * Modified by R. Butler
 */

#include "usc_sys.h"


VOID usc_init()
{

#if defined(MULTIMAX)

	usc_multimax_timer = timer_init();
	usc_MD_rollover_val = (usc_time_t) ((1<<usc_MD_timer_size)-1);

#endif


#if defined(SYMMETRY) || defined(SYMMETRY_PTX)

	unsigned long roll;

	usclk_init();

	roll = 1 << (usc_MD_timer_size-1);
	usc_MD_rollover_val = (usc_time_t) (roll + roll - 1);

#endif


#if defined(TC_2000) || defined(TC_2000_TCMP)

	unsigned long roll;

	roll = 1 << (usc_MD_timer_size-1);
	usc_MD_rollover_val = (usc_time_t) (roll + roll - 1);

#endif


#if defined(IPSC860)

	esize_t hwtime;
	unsigned long ustime;

	hwtime.shigh = hwtime.slow = ~0x0;
        hwtime.shigh = (hwtime.shigh & 0x7) << (sizeof(long)*8-3);
        hwtime.slow = ((hwtime.slow >> 3) & ~(0x7 << (sizeof(long)*8-3)))
				| hwtime.shigh;
        ustime = (unsigned long)hwtime.slow * 0.8;
	usc_MD_rollover_val = (usc_time_t) ustime; 

#endif

#if defined(FX2800)  ||  defined(FX2800_SWITCH)

    struct hrcval temptime;
    unsigned long roll;

    hrcstamp(&temptime);
    roll = 1 << ((sizeof(usc_time_t) * 8) - 1);
    usc_MD_rollover_val = (usc_time_t) (roll + roll - 1);

#endif


#if defined(SUN) || defined(HP) || \
    defined(BALANCE) || \
    defined(RS6000)  ||  defined(IBM3090) || \
    defined(NEXT) || defined(TITAN) || defined(GP_1000) || \
    defined(KSR)  || \
    defined(SGI)  || defined(FX8)

	struct timeval tp;
	struct timezone tzp;
	unsigned long roll;

	gettimeofday(&tp,&tzp);

	roll = 1 << ((sizeof(usc_time_t)*8)-1);
	roll = roll + roll - 1;
	usc_MD_rollover_val = (usc_time_t) (roll / 1000000);

#endif

}



usc_time_t usc_MD_clock()
{

#if defined(TC_2000) || defined(TC_2000_TCMP)

    struct 
    {
	unsigned long hi;
	unsigned long low;
    } usclock;

    get64bitclock(&usclock);
    return((usc_time_t)usclock.low);

#endif


#if defined(IPSC860)

	esize_t hwtime;
	unsigned long ustime;

	hwclock(&hwtime);
        hwtime.shigh = (hwtime.shigh & 0x7) << (sizeof(long)*8-3);
        hwtime.slow = ((hwtime.slow >> 3) & ~(0x7 << (sizeof(long)*8-3)))
				| hwtime.shigh;
        ustime = (unsigned long)hwtime.slow * 0.8;
	return( (usc_time_t) ustime);

#endif

#if defined(FX2800)  ||  defined(FX2800_SWITCH)

    struct hrcval temptime;

    hrcstamp(&temptime);
    return ((usc_time_t) (temptime.hv_low * 10));

#endif


#if defined(SUN) || defined(HP) || \
    defined(BALANCE) || \
    defined(RS6000) || defined(IBM3090) || \
    defined(NEXT) || defined(TITAN) || defined(TC1000) || \
    defined(KSR)  || \
    defined(SGI) || defined(FX8)

	unsigned long ustime;
	struct timeval tp;
	struct timezone tzp;

	gettimeofday(&tp,&tzp);
	ustime = (unsigned long) tp.tv_sec;
	ustime = ustime % usc_MD_rollover_val;
	ustime = (ustime * 1000000) + (unsigned long) tp.tv_usec;

	return((usc_time_t) ustime);

#endif

}
SHAR_EOF
if test -f 'usc_sys.h'
then
	echo shar: over-writing existing file "'usc_sys.h'"
fi
cat << \SHAR_EOF > 'usc_sys.h'
/*
 * USC_MAIN.H  (Private header file for the Microsecond Clock package)
 *
 * Written by:  Arun Nanda    (07/17/91)
 * Modified by R. Butler
 *
 * The following machine-specific libraries need to be linked in
 * with the application using the UST functions:
 *
 */


#include "usc.h"


#if defined(MULTIMAX)

#    include <parallel.h>
#    define usc_MD_timer_size  (sizeof(unsigned)*8)
     unsigned *usc_multimax_timer;

#endif


#if defined(SYMMETRY) || defined(SYMMETRY_PTX)

#    define usc_MD_timer_size  (sizeof(usclk_t)*8)

#endif


#if defined(TC_2000) || defined(TC_2000_TCMP)

#    define usc_MD_timer_size  (sizeof(unsigned long)*8)

#endif


#if defined(IPSC860)

#    if defined (DELTA)
#        include <mesh.h>
#    else
#        include <cube.h>
#    endif
#    define usc_MD_timer_size ((sizeof(long)*8)+3)
#    define usc_MD_ticks_per_usec (HWHZ/1000000)

#endif


#if defined(FX2800)  ||  defined(FX2800_SWITCH)
#       include <sys/time.h>
#endif

#if defined(SUN) || defined(DEC5000) || defined(HP) \
    || defined(BALANCE) \
    || defined(IBM3090) || defined(RS6000) \
    || defined(NEXT) || defined(TITAN) || defined(GP_1000) \
    || defined(KSR) \
    || defined(SGI) || defined(FX8)

#	include <sys/time.h>

#endif


usc_time_t usc_MD_rollover_val = 0;

SHAR_EOF
if test ! -d 'usctest'
then
	mkdir 'usctest'
fi
cd 'usctest'
if test -f 'usctest.c'
then
	echo shar: over-writing existing file "'usctest.c'"
fi
cat << \SHAR_EOF > 'usctest.c'
#include <stdio.h>
#include "../alog_usc.h"

#if defined(CAP2_HOST)
#include <chost.c7.h>
host_main()
#else
#if defined(CAP2_CELL)
cell_main()
#else
main()
#endif
#endif
{
	usc_init();
	printf("\nRollover Value = %lu\n", usc_rollover_val());

	printf("\n\nShort Loop Test:\n");
	printf("================\n\n");
	short_loop();

	printf("\n\nInfinite Loop Test (measures 5 second intervals):\n");
	printf("=================================================\n");
	printf("(***** Type ^C to terminate this test *****)\n\n");
	infinite_loop();
}


short_loop()
{
	usc_time_t t1, t2, t3;
	usc_time_t t[100];
	int i;

	for (i=0; i<100; i++)
		t[i] = usc_clock();
	for (i=0; i<100; i++)
		printf("Clock Reading %2d:  %lu\n", i+1, t[i]);

	printf("\nThree additional readings...just for the heck of it\n");
	t1 = usc_clock();
	t2 = usc_clock();
	t3 = usc_clock();
	printf("time1 = %lu,  time2 = %lu,  time3 = %lu\n", t1, t2, t3);
}


infinite_loop()
{
	usc_time_t t1, t2;
	int i,j;

	for(j=0;j < 10; j++)
	{
		t1 = usc_clock();
#if !defined(CAP2_CELL)

		sleep(5);
#else
		{
		double dgettime();
		double dinit;

		dinit = dgettime();
		while (dgettime() < dinit + 5)
		    ;
		}
#endif

		t2 = usc_clock();
		printf("Start_time = %lu    End_time = %lu\n\n", t1, t2);
		printf("---> Interval = %lu microsecs <---\n\n", t2-t1);
	}
}
SHAR_EOF
if test -f 'capcell.c'
then
	echo shar: over-writing existing file "'capcell.c'"
fi
cat << \SHAR_EOF > 'capcell.c'
#include <stdio.h>
#include "../usc.h"
#include "ccell.c7.h"
#include "../alog.h"

#define SLOOP_START 1
#define SLOOP_END   2
#define ILOOP_START 3
#define ILOOP_END   4

cell_main()
{
	int cellid;
	
	cellid = getcid();

	if (cellid == 0)
	{
	    ALOG_MASTER(0,ALOG_TRUNCATE);
	}
	else
	{
	    ALOG_SETUP(cellid,ALOG_TRUNCATE);
	}

	usc_init();
	printf("\nRollover Value = %lu\n", usc_rollover_val());

	printf("\n\nShort Loop Test:\n");
	printf("================\n\n");
	ALOG_LOG(cellid,SLOOP_START,0,"");
	short_loop();
	ALOG_LOG(cellid,SLOOP_END,0,"");

	printf("\n\nInfinite Loop Test (measures 5 second intervals):\n");
	printf("=================================================\n");
	printf("(***** Type ^C to terminate this test *****)\n\n");
	ALOG_LOG(cellid,ILOOP_START,0,"");
	infinite_loop();
	ALOG_LOG(cellid,ILOOP_END,0,"");

	ALOG_OUTPUT;
}


short_loop()
{
	usc_time_t t1, t2, t3;
	usc_time_t t[100];
	int i;

	for (i=0; i<100; i++)
		t[i] = usc_clock();
	for (i=0; i<100; i++)
		printf("Clock Reading %2d:  %lu\n", i+1, t[i]);

	printf("\nThree additional readings...just for the heck of it\n");
	t1 = usc_clock();
	t2 = usc_clock();
	t3 = usc_clock();
	printf("time1 = %lu,  time2 = %lu,  time3 = %lu\n", t1, t2, t3);
}


infinite_loop()
{
	usc_time_t t1, t2;
	int i,j;

	for(j=0;j < 10; j++)
	{
		t1 = usc_clock();
#if !defined(CAP2_CELL)

		sleep(5);
#else
		{
		double dgettime();
		double dinit;

		dinit = dgettime();
		while (dgettime() < dinit + 5)
		    ;
		}
#endif

		t2 = usc_clock();
		printf("Start_time = %lu    End_time = %lu\n\n", t1, t2);
		printf("---> Interval = %lu microsecs <---\n\n", t2-t1);
	}
}
SHAR_EOF
if test -f 'usctest2.c'
then
	echo shar: over-writing existing file "'usctest2.c'"
fi
cat << \SHAR_EOF > 'usctest2.c'
#include <stdio.h>
#include "../usc.h"

#if defined(CAP2_HOST)
#include <chost.c7.h>
host_main()
#else
#if defined(CAP2_CELL)
cell_main()
#else
main()
#endif
#endif
{
	cconfxy(64,1);
	ccreat("capcell",30,NULL);
}

SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
P4_HOME_DIR = ../..
LIBDIR = $(P4_HOME_DIR)/lib
LIBS =  $(MDEP_LIBS)


default:

usctest: usctest.o
	$(CC) $(CFLAGS) -o usctest usctest.o ../usc.o $(LIBS)

usctest2: usctest2.o
	$(CC) $(CFLAGS) -o usctest2 usctest2.o ../usc.o $(LIBS)

capcell: capcell.o
	$(CC) $(CFLAGS) -o capcell capcell.o ../usc.o $(LIBS)

realclean:	clean
	$(RM) -f *~ *.o usctest usctest2 capcell Makefile

SHAR_EOF
cd ..
cd ..
if test ! -d 'util'
then
	mkdir 'util'
fi
cd 'util'
if test -f 'defs.all'
then
	echo shar: over-writing existing file "'defs.all'"
fi
cat << \SHAR_EOF > 'defs.all'
# top level makefile for P4

#   BEGIN SUN
# P4 makefile for Sun
MACHINE = SUN
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END SUN

#   BEGIN HP
# P4 makefile for HP
MACHINE = HP
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = true
MDEP_LIBS = -lV3
# -lV3 is for sighold, etc.
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END HP

#   BEGIN DEC5000
# P4 makefile for DEC 5000
# DEC looks like a Sun workstation
MACHINE = DEC5000
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END DEC5000

#   BEGIN RS6000
# P4 makefile for IBM RS/6000
MACHINE = RS6000
AR = ar ruv
FC = xlf
CLINKER = cc 
RANLIB = true
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = -qextname 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END RS6000

#   BEGIN IBM3090
# P4 makefile for IBM 3090 running AIX
MACHINE = IBM3090
AR = ar ruv
CLINKER = cc 
FC = fvs
RANLIB = true
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END IBM3090

#   BEGIN TITAN
# P4 makefile for Ardent Titan
MACHINE = TITAN
AR = ar ruv
FC = fc
CLINKER = cc 
RANLIB = true
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END TITAN

#   BEGIN SGI
# P4 makefile for Silicon Graphics
MACHINE = SGI
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = true
MDEP_LIBS = -lsun
MDEP_CFLAGS = -cckr
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END SGI

#   BEGIN NEXT
# P4 makefile for NeXT
MACHINE = NEXT
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS =
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END NEXT

#   BEGIN KSR
# P4 makefile for Kendall Square
MACHINE = KSR
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = true
MDEP_LIBS = -lrpc
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END KSR

#   BEGIN FX8
# P4 makefile for Alliant FX/8
MACHINE = FX8
AR = ar ruv
FC = fortran
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS = 
MDEP_CFLAGS = -nc
MDEP_FFLAGS = -nc
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END FX8

#   BEGIN FX2800
# P4 makefile for Alliant FX/800 and FX/2800
MACHINE = FX2800
AR = ar ruv
FC = fortran
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS = -lalliant
MDEP_CFLAGS = -nc
MDEP_FFLAGS = -nc
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END FX2800

#   BEGIN FX2800_SWITCH
# P4 makefile for Alliant FX/800 and  Fx/2800 with HiPPI Switch
MACHINE = FX2800_SWITCH
AR = ar ruv
FC = fortran
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS = /usr/local/lib/libsw.a -lalliant
MDEP_CFLAGS = -nc -I/usr/local/include
MDEP_FFLAGS = -nc
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END FX2800_SWITCH

#   BEGIN DELTA
# P4 makefile for Intel DELTA
MACHINE = DELTA
FILE_LINK = ln -s 
AR = ar860 r
FC = if77
CLINKER = icc 
RANLIB = true
MDEP_LIBS = -lsocknode 
MDEP_CFLAGS = -node -i860
MDEP_FFLAGS = -node -i860
P4_OBJ	= $(P4_COMM_OBJ) $(P4_SOCK_OBJ)
CC = icc
#   END DELTA

#   BEGIN IPSC860
# P4 makefile for Intel iPSC860 node with Portland Group Compilers
MACHINE = IPSC860
FILE_LINK = ln -s 
AR = ar860 r
FC = if77
CLINKER = icc 
RANLIB = true
#MDEP_LIBS = -lnode
MDEP_CFLAGS = -O3 -Mvect -node
MDEP_FFLAGS = -O3 -Mvect -node
P4_OBJ	= $(P4_COMM_OBJ)
CC = icc
#   END IPSC860

#   BEGIN CM5
# P4 makefile for Thinking Machine's CM-5
MACHINE = CM5
FILE_LINK = ln -s 
AR = ar ruv
FC = f77
CLINKER = cmmd-ld -comp cc 
FLINKER = cmmd-ld -comp f77
RANLIB = ranlib
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ	= $(P4_COMM_OBJ)
CC = cc
#   END CM5

#   BEGIN IPSC860_SOCKETS
# P4 makefile for Intel iPSC860 node with sockets and Portland Group Compilers
MACHINE = IPSC860_SOCKETS
FILE_LINK = ln -s 
AR = ar860 r
FC = if77
CLINKER = icc 
RANLIB = true
MDEP_LIBS = -lsocknode 
MDEP_CFLAGS = -O3 -Mvect -node
MDEP_FFLAGS = -O3 -Mvect -node
P4_OBJ	= $(P4_COMM_OBJ) $(P4_SOCK_OBJ)
CC = icc
#   END IPSC860_SOCKETS

#   BEGIN BALANCE
# P4 makefile for Sequent Balance
MACHINE = BALANCE
FILE_LINK = ln 
FC = fortran
AR = ar ruv
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS= -lpps -lseq
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ	= $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
P = &
#   END BALANCE

#   BEGIN SYMMETRY
# P4 makefile for Sequent Symmetry
MACHINE = SYMMETRY
FILE_LINK = ln 
FC = fortran
AR = ar ruv
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS= -lpps -lseq
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ	= $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
P = &
#   END SYMMETRY

#   BEGIN SYMMETRY_PTX
#   P4 makefile for Sequent Symmetry PTX
#
# The '-Wc,-seq' option is needed to recognize
# variable types "shared" or "private", etc.
#
MACHINE = SYMMETRY_PTX
FILE_LINK = ln
FC = fortran
AR = ar ruv
CLINKER = cc 
RANLIB = true
MDEP_LIBS= -lpps -lseq -lrpc -lsocket -linet -lnsl
MDEP_CFLAGS = -Wc,-seq
P4_OBJ  = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
# P4_OBJ  = $(P4_COMM_OBJ) $(P4_SHMEM_OBJ)
P = &
#   END SYMMETRY_PTX

#   BEGIN MULTIMAX
# P4 makefile for Encore Multimax
MACHINE = MULTIMAX
FILE_LINK = ln 
FC = f77
AR = ar ruv
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS= -lpp
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ	= $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END MULTIMAX

#   BEGIN CRAY
# P4 makefile for Cray
MACHINE = CRAY
AR = ar ruv
CC = scc
FC = cf77
CLINKER = scc 
RANLIB = true
MDEP_LIBS = -lrpc,net
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END CRAY

#   BEGIN GP_1000
# P4 makefile for BBN Butterfly GP-1000
MACHINE = GP_1000
FILE_LINK = ln 
AR = ar ruv
FC = f77
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS= -lus
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
P = $(PARALLEL)
#   END GP_1000

#   BEGIN TC_2000
# P4 makefile for BBN Butterfly TC-2000
MACHINE = TC_2000
FILE_LINK = ln 
FC = f77
AR = ar ruv
CLINKER = cc 
RANLIB = ranlib
MDEP_LIBS= -lus
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
P = $(PARALLEL)
#   END TC_2000

#   BEGIN TC_2000_TCMP
# P4 makefile for BBN Butterfly TC_2000 with TCMP message-passing library
MACHINE = TC_2000_TCMP
FILE_LINK = ln 
FC = f77
AR = ar ruv
P = $(PARALLEL)
CLINKER = cc 
RANLIB = ranlib
#TCMP_LIBDIR = /Net/sparky/sparky1/lusk/lepido/tcmp
TCMP_LIBDIR = /usr/bbnm/tcmp
TCMPLIB = $(TCMP_LIBDIR)/libtcmpnew.a   # or see system administrator
MDEP_LIBS =  $(TCMPLIB) -lus
MDEP_CFLAGS = 
MDEP_FFLAGS = 
P4_OBJ = $(P4_COMM_OBJ) $(P4_SOCK_OBJ) $(P4_SHMEM_OBJ)
#   END TC_2000_TCMP

#   BEGIN COMMON

RM = /bin/rm

CFLAGS = -g -I$(INCLUDEDIR) $(MDEP_CFLAGS) $(USER_CFLAGS)

FFLAGS = -g -I$(FORTLIBDIR) $(MDEP_FFLAGS)

.SUFFIXES:	.o .c .f .h

.c.o:$(P)
	$(CC) $(CFLAGS) -c $*.c 

.f.o:$(P)
	$(FC) $(FFLAGS) -c $*.f 

dummy:	default

clean:
	$(RM) -f *~ *.o *.bak tags TAGS core

#   END COMMON

SHAR_EOF
if test -f 'install.awk'
then
	echo shar: over-writing existing file "'install.awk'"
fi
cat << \SHAR_EOF > 'install.awk'
# awk script for editing prototype p4 makefile

/END/   	{ include = "off" }
include == "on"	
/BEGIN/ 	{ if (($3 == machine) || ($3 == "COMMON") ) include = "on" }

SHAR_EOF
if test -f 'machines'
then
	echo shar: over-writing existing file "'machines'"
fi
cat << \SHAR_EOF > 'machines'
    SUN
    HP
    DEC5000
    RS6000
    IBM3090
    TITAN
    SGI
    NEXT
    FX8
    FX2800
    FX2800_SWITCH
    IPSC860
    IPSC860_SOCKETS
    DELTA
    BALANCE
    SYMMETRY
    SYMMETRY_PTX
    MULTIMAX
    CRAY
    GP_1000
    TC_2000
    TC_2000_TCMP
    KSR
    CM5
SHAR_EOF
if test -f 'makefile.proto'
then
	echo shar: over-writing existing file "'makefile.proto'"
fi
cat << \SHAR_EOF > 'makefile.proto'
# prototype makefile for util directory of P4

RM = rm

defs.MD: install.awk defs.all
	@if grep $(MACHINE) machines > /dev/null ;\
	then \
	echo making machine-dependent definitions for $(MACHINE) ;\
	awk -f install.awk machine=$(MACHINE) defs.all > defs.MD ;\
	else \
	echo cannot install for $(MACHINE), valid machines are: ;\
	cat machines ;\
	exit 1 ;\
	fi

default:

all:

realclean:
	$(RM) -f Makefile defs.MD *~ *.o
SHAR_EOF
cd ..
if test -f 'DISCLAIMER'
then
	echo shar: over-writing existing file "'DISCLAIMER'"
fi
cat << \SHAR_EOF > 'DISCLAIMER'


This code was written in the hope that it might be useful to others as well
as to ourselves, but is not guaranteed in any way.  Anyone is free to copy it
and modify it to suit their own purposes.

We would appreciate comments, bug reports, experiences, and enhancement
requests.

SHAR_EOF
cd ..
#	End of shell archive
exit 0
