/*********************************************************************

SWAGES                                                  */ global vars
Version = '1.0 alpha   '                                           ;/*

(c) by Matthias Scheutz

GridServer

Last modified: 06-21-03

SYNOPSIS:

The SWAGES Grid Server provides a grid computing facility to support the 
automatic sequencing of experiments.  It takes as its first optional argument
a hosts file, and as its second a file that will be compiled (e.g., this 
can be used to configure the server or start the server in non-interactive
mode).  By default, the server will start with a web interface (in interactive 
mode).

Experiments can be started by passing an additional 4th argument to the
simulation list: [simparams entitylist resourcelist serverparams] which contains
specific information for the server.

*********************************************************************/


;;; Increase popmemlim to reduce garbage collection time.
max(popmemlim, 128000000) -> popmemlim;

lconstant this_dir = sys_fname_path(popfilename);

;;; default global, can be changed in the experiment setup file
global vars

    ;;; the server configuration file, will be compiled upon startup
    serverconfig = 'swages.cfg',
    ES,                        ;;; this will hold the instance of the experiment server

    ;;; various paths to supporting scripts and directories----------------------------     

    ;;; path to the simulation files (which will be copied over it necessary)    
    runpathsolaris = '/afs/nd.edu/user40/airolab/projects/swages/',
    runpathlinux = '/afs/nd.edu/user40/airolab/projects/swages/',

    ;;; the path to the ping script including all necessary parameters
    ;;; plus the args, need to have spaces in the front and the back 
    pingpathsolaris = '/usr/sbin/ping',
    pingpathlinux = '/bin/ping',
    pingargssolaris = ' ',
    pingargslinux = '-c 1 -q -w 10',
    pingpath,                   ;;; this will be set upon startup

    ;;; where the hostcheck files will be stored and what the timeout is for checking hosts
    testhostfilescratch = '/tmp',
    testhostscriptname = 'checkhosts',
    testhostscript = '',        ;;; will hold the filepath to the script
    TESTHOSTTIMEOUT = 15,
    generateTESTHOSTSCRIPT = true,

    ;;; path to the remote shell command to be used to log on and start experiments
    ;;; and the args for ssh, needs to have spaces in the front and the back
    sshpathsolaris = '/afs/nd.edu/sun4x_58/usr/afsws/bin/rsh',
    sshpathlinux = '/afs/nd.edu/user40/airolab/projects/swages/ssh',
    sshargssolaris = '',
    sshargslinux = '-1 -A -x -n',    
    SSHTIMEOUT = 30,
    SSHTIMEOUT2 = 18000000,
    SCHEDULETIME = 5,
    sshpath,                   ;;; this will be set upon startup

    ;;; path to the remote copy command to be used in conjunction with copying over files
    ;;; this will use ssh with protocol 1
    ;;; and the args for scp, needs to have spaces in the front and the back
    scppathsolaris = '/bin/scp',
    scppathlinux = '/afs/nd.edu/user40/airolab/projects/swages/scp',
    ;;;scpargssolaris = '-oProtocol=1 -oAFSTokenPassing=yes',
    scpargssolaris = '-oProtocol=1',
    ;;;scpargslinux = ' -S ' >< sshpathlinux >< ' -oProtocol=1 -oAFSTokenPassing=yes ',
    scpargslinux = ' -S ' >< sshpathlinux >< ' -oProtocol=1 ',
    SCPTIMEOUT = 600,
    scppath,                   ;;; this will be set upon startup

    ;;; the operating system of the server
    serverOS = 'linux',        ;;; OS of the server, make sure this is a string

    ;;; various file names for output and simulation files -----------------------------
    ;;; standard file with remote host names and the # of runs through it before quitting
    hostsfile = 'ndsuns.lst',

    ;;; wait in seconds in case all hosts have been tried out
    WAITINGFORHOSTS = 19,
    ;;; if this is false, hosts will not be checked
    mustCHECKHOSTS = true,
    CHECKHOSTS = mustCHECKHOSTS,
    numclientswaiting = 0,      ;;; used for the number of clients waiting for a host
    pnumclientswaiting = 0,     ;;; used for the number of clients with priority waiting for a host

    HOSTCHECKFREQ = 180,       ;;; frequency of checking a participating host in sec
    WATCHDOGINTERVAL = 300,    ;;; frequency of checking clients in sec. for the watchdog
    MAXMIGRATIONTIME = 1800,   ;;; the max. time for process to migrate (i.e., before the client gets reset)
    watchdogRUNNING = false,   ;;; for bookkeeping

    ;;; the domain of the server and the port number to connect to
    myhost = 'kaundka.cse.nd.edu',
    portnumber = 10007,
    
    ;;; this is used to determine whether the state of the client should be saved periodically
    useTIMER = false,
    
    ;;; this determines whether the server runs in batch mode or in interactive mode
    ;;; in batch mode, the server needs to be either started by passing an experiment files as an
    ;;; argument, or the "setup(hostfiles,expfiles)" needs to be invoked explicitly
    ;;; in interactive mode, on the other hand, the web server will be started and experiments
    ;;; can be monitored, altered, view, and submitted through the WWW interface
    ;;; (this requires the "webserver.p" file)
    interactiveMODE = true,
    ignoreUNKNOWNKEYWORD = true,

    ;;; TODO: implement a state saving mechanism for the server too
    ;;; whether the server should save its state
    saveSERVERSTATE = false,

    ;;; globals for the web server (the port, the site layout and main entry point)
    comm_port = 10001,
    serverweb = [],
    procedure webrequest,
    ;;; this is the line-feed string for printing
    ;;; for html this should be changed to something like '<br>\n'
    lfchar = '\n',
;

;;; constants for the server
lconstant
     ;;; the ID of the server
     SID = 0,
    ;;; for debugging: print out lots of comments
    verbose = true,
;

;;; -------------------------------------------------------------------
;;; include the unix_sockets, datasockets, and threads packages
uses unix_sockets;
;;;include include unix_sockets.ph;

compile(this_dir dir_>< 'swsclasses.p');
compile(this_dir dir_>< 'util.p');
compile(this_dir dir_>< 'threads.p');
compile(this_dir dir_>< 'datasockets.p');

;;; ----------------------------------------------------------------------------------------------------------

;;; this will set up various paths
define setuppaths();
    valof(consword('pingpath' >< serverOS))><' '>< valof(consword('pingargs' >< serverOS))><' '-> pingpath;
    valof(consword('sshpath'><serverOS))><' '>< valof(consword('sshargs'><serverOS))><' '-> sshpath;
    valof(consword('scppath'><serverOS))><' '>< valof(consword('scpargs'><serverOS))><' '-> scppath;
enddefine;


;;; new uninterruptable version of delete that does not follow symbolic links
define deleteit(filename);
    dlocal pop_asts_enabled = false,pop_file_versions = false;
    sysdelete(filename,false)
enddefine;    


;;; this will generate a script for checking the host depending on various systems settings
;;; the first argument it the name of the remote host, the second is the scriptpath on the remote host
;;; the output from the remote host will be placed in a file named after the host 
;;; in the directory where the script resides with the
define generatetesthostscript();
    testhostfilescratch dir_>< testhostscriptname -> testhostscript;
    deleteit(testhostscript)->;    
    dlocal cucharout = discout(testhostscript);
    npr('#!/bin/sh\nif '>< pingpath >< ' $1 > /dev/null 2>&1 ; then');
    npr('  ' >< sshpath >< ' $1 $2 > $3 2>&1\nelse\n  echo down > $3\nfi');
    ;;; close the file
    cucharout(termin);
    ;;; make the file executable
    33261 -> sysfilemode(testhostscript);
enddefine;
    
;;; Returns process exit status of child if completed, or false if kill failed, or true if kill succeeded
define sysobeynew(args,sleeptime /*,withfilecheck,deletefile*/)/*->success*/;
    ;;; turn off interrupts
    dlocal pop_asts_enabled = false;
    lvars child_pid, withfilecheck = false, deletefile = false;

    ;;; check if we have four arguments
    if isboolean(sleeptime) then
	(args,sleeptime) -> (args,sleeptime,withfilecheck,deletefile);
    endif;
    
    ;;; first discard any waiting message
    receive_thread()->;
    ;;; we will never wait on the child...
    if sys_vfork(false)->>child_pid then
	;;; wait for "sleeptime" seconds, if the child has not reported back by then, terminate it
	while sleeptime > 0 do
	    ;;; check if the file "withfilecheck" exists, and if so, return
	    if withfilecheck and sys_file_exists(withfilecheck) then
		;;; the file exists, so delete it and try to terminate the child (just in case it is still alive...)
    		deletefile and deleteit(withfilecheck)->;		
		return(not(syskill(child_pid)));
	    endif;
	    sleep_thread(1);
	    sleeptime - 1 -> sleeptime;
	endwhile;
	not(syskill(child_pid))
    else
    	sysexecute('/bin/sh', ['/bin/sh' '-c' ^args], false)
    endif
enddefine;

define dosysgarbage();
    dlocal pop_asts_enabled = false;
    npr('Garbage...');
    sysgarbage();
enddefine;

;;; this new line_repeater works just like to old one except that it now takes the process_id
;;; of the child process, which it will kill if it does not respond within "timeout" seconds
;;; furthermore, an updater is defined that will kill the child as well, it takes no arguments
;;; e.g., if l is a newline_repeater, then "->l()" will kill the child process and close the device
define newline_repeater(child_pid, file, lim, timeout);
    lvars file, dev, buffer;
    
    define lconstant Get_line(childpid, dev, buffer, lim, timeout) with_props line_repeater;
        lvars dev, buffer, lim, n;
	repeat timeout times
	    sys_input_waiting(dev) -> n;
	    if not(n) then
		;;; no bytes are available, so wait
	    	sleep_thread(1);
	    else
		;;; otherwise read in a line
                fast_sysread(dev, 1, buffer, lim) -> n;
		;;; if 0 is returned, then file was closed
                if n == 0 then
		    ;;; close the device
                    ;;;unless systrmdev(dev) or dev == popdevin then
                    sysclose(dev);
                    ;;;;endunless;
		    ;;; we're done, to be safe kill the child process
		    if not(syskill(childpid)) and verbose then
			npr('Could not kill child process in newline_repeater ' >< childpid);
		    endif;
                    ;;;dosysgarbage();
                    return(termin);
                else
                    return(substring(1, n fi_- 1, buffer));
                endif;
	    endif;
	endrepeat;
	;;; we timed out, so kill the child process
	if not(syskill(childpid)) and verbose then
	    npr('Could not kill child process in newline_repeater ' >< childpid);
	endif;
	sysclose(dev);
        ;;;dosysgarbage();
	return(termin);
    enddefine;
    
    if isdevice(file) then
        file
        ;;; Should also check that the device was opened in "line" mode
    else
        sysopen(file, 0, "line", `N`)
    endif -> dev;
    
    lim or 80 -> lim;
    if isstring(lim) then
        lim -> buffer;
        datalength(buffer) -> lim;
    elseif isinteger(lim) then                
	device_init_buffer(dev, lim + 1) -> buffer
    else
        mishap(file, lim, 2, 'STRING (buffer) OR INTEGER (buffer size) NEEDED')
    endif;

    ;;; assign a closure that will kill the child process as updater 
    procedure(childpid,d);
	sysclose(d);
	if not(syskill(childpid)) and verbose then
	    npr('Could not kill child process in newline_repeater ' >< childpid);
	endif;
        ;;;dosysgarbage();
    endprocedure(%child_pid,dev%) -> updater(Get_line);

    Get_line(% child_pid, dev, buffer, lim, timeout %)
enddefine;

;;; the new pipein command for line mode, returns the child_pid and input stream
define newpipein(command, args) -> (child_pid,din);
    lvars din, dout, result, child_pid, mesg;
    ;;; Make the pipe.
    syspipe("line") -> (dout, din);
    if sys_vfork(false) ->> child_pid then		;;; false = don't want to wait for child
	;;; parent - get characters from pipe
	sysclose(dout);
    else
	;;; child - execute the command to produce the output
	sysclose(din);
	dout -> popdevout;
	dout -> popdeverr;			;;; ensure you get error messages
	sysexecute(command, args, false);
    endif;
    ;;;dosysgarbage();
enddefine;

;;; ----------- the server methods  --------------------------------------

;;; puts a print string on the stack
define :method printhostinfo(host:simhost);
    '*** HOST INFORMATION ***' >< lfchar ><
    'Name:          ' >< host.name >< lfchar ><
    'Path to sim:   ' >< host.simpath >< lfchar ><
    'Path to pop:   ' >< host.poppath >< lfchar ><
    'Scratch space: ' >< host.scratch >< lfchar ><
    'IP:            ' >< host.IP >< lfchar ><
    'OS:            ' >< host.OS >< lfchar ><
    'Cpusonhost:    ' >< host.cpusonhost >< lfchar ><
    'Avail.crit:    ' >< host.availcrit >< lfchar ><
    'Load:          ' >< host.loadonhost >< lfchar ><
    'Users:         ' >< host.users >< lfchar ><
    'Console user:  ' >< (if host.consoleuser then 'yes' else 'no' endif) >< lfchar ><
    'Console idle:  ' >< (if host.idleuser then 'yes' else 'no' endif) >< lfchar ><
    'Cpus used:     ' >< host.cpusused >< lfchar ><
    'Available:     ' >< (if host.available then 'yes' else 'no' endif) >< lfchar ><
    'Last checked:  ' >< host.lastchecked >< lfchar ><
    'Sim clients:   ' >< (if host.clients then host.clients else 'none' endif) >< lfchar ><
    '------------------------------' >< lfchar
enddefine;

;;; this updates the host information
define :method updatehost(host:simhost);
    dlocal interrupt = identfn, pop_status, pop_file_versions = false;

    ;;; for now set the availability of the host to false
    false -> host.available;
    if verbose then npr('Checking host ' >< host.name); endif;
	 
    ;;; remove any old test host file...
    lvars testhostfile = testhostfilescratch dir_>< host.name;
    deleteit(testhostfile)->;
    ;;; run script (eventually with trap, so we get notified when it completed and won't have to wait till it times out
    lvars ret = sysobeynew(testhostscript >< ' ' >< host.name >< ' ' >< host.simpath dir_>< 'test_host' >< ' ' >< testhostfile,TESTHOSTTIMEOUT);
    ;;; now read in the info and return if the file does not exist (then the script musthave gotten stuck at some point during ping)
    returnunless(sys_file_exists(testhostfile));
    ;;; otherwise we got something, so open it with a line repeater
    lvars line_rep = line_repeater(testhostfile,255),
	 load = line_rep(), 
         user, cpus, console, idle;
    ;;; if it is different from 'down', then we got some info...
    if load /= 'down' then
	unless load then
	    ;;;termin -> line_rep();
	    return;
	endunless;
	while(load /= 'test_host') do
	    if verbose then npr(host.name >< ' message: ' >< load); endif;
	    if load = 'timeout' then
		if verbose then npr('updatehost timed out') endif;
	    	;;;termin -> line_rep();
		return;
	    elseif load = termin then
		if verbose then npr('updatehost received <termin>') endif;
	    	;;;termin -> line_rep();
		return;
	    endif;
	    line_rep() -> load;
	endwhile;
	line_rep() -> load;
	;;;npr(host.name >< ' load: ' >< load);sysflush(pop_charout_device);
	if load = 'timeout' then
	    if verbose then npr('updatehost timed out') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif load = termin then
	    if verbose then npr('updatehost received <termin>') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif isstring(load) then
	    strnumber(load) -> host.loadonhost;
	else
	    if verbose then npr('updatehost: load not string: ' >< load) endif;
	    ;;;termin -> line_rep();
	    return;
	endif;
	;;; 2nd line is # of users
	line_rep() -> user;
	;;;npr(host.name >< ' user: ' >< user);sysflush(pop_charout_device);
	if user = 'timeout' then
	    if verbose then npr('updatehost timed out') endif;
	    ;;;termin -> line_rep();	    
	    return;
	elseif user = termin then
	    if verbose then npr('updatehost received <termin>') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif isstring(user) then
	    strnumber(user) -> host.users;
	else
	    if verbose then npr('updatehost: user not string: ' >< user) endif;
	    ;;;termin -> line_rep();
	    return;
	endif;
	;;; 3rd line is # of cpus used
	line_rep() -> cpus;
	;;;npr(host.name >< ' cpus: ' >< cpus);sysflush(pop_charout_device);
	if cpus = 'timeout' then
	    if verbose then npr('updatehost timed out') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif cpus = termin then
	    if verbose then npr('updatehost received <termin>') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif isstring(cpus) then
	    strnumber(cpus) -> host.cpusused;
	else
	    if verbose then npr('updatehost: cpus not string: ' >< cpus) endif;
	    ;;;termin -> line_rep();
	    return;
	endif;
	;;; 4th line is console user
	line_rep() -> console;
	;;;npr(host.name >< ' console: '><console);sysflush(pop_charout_device);
	if console = 'timeout' then
	    if verbose then npr('updatehost timed out') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif console = termin then
	    if verbose then npr('updatehost received <termin>') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif isstring(console) then
	    valof(consword(console)) -> host.consoleuser;
	else
	    if verbose then npr('updatehost: console not string: ' >< console) endif;
	    ;;;termin -> line_rep();
	    return;
	endif;
	;;; 5th line is idle status of console user, true if console user 
	;;; is idle, false if not OR no console user
	line_rep() -> idle;
	if idle = 'timeout' then
	    if verbose then npr('updatehost timed out') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif idle = termin then
	    if verbose then npr('updatehost received <termin>') endif;
	    ;;;termin -> line_rep();
	    return;
	elseif isstring(idle) then
	    valof(consword(idle)) -> host.idleuser;
	else
	    if verbose then npr('updatehost: idle not string: ' >< idle) endif;
	    ;;;termin -> line_rep();
	    return;
	endif;
	;;;termin -> line_rep();	    
	sysdaytime() -> host.lastchecked;
	;;; mark the host as unavailable if all cpus are used up for simulations
	if host.cpusused >= host.cpusonhost then
		false -> host.available;
	;;; now check availability of host
	elseif host.availcrit then
	    lvars item, currenttime = substring(12,8,sysdaytime());
	    for item in host.availcrit do
	    	lvars level = item(1),
		     timeok = if length(item) = 3 then
				  ;;; PWS: I don't think w_t_i() works correctly
		                  within_time_interval(currenttime,item(2),item(3))
		              elseif length(item) = 2 then
			          item(2)
		              else true endif;
            	;;; check if anybody is on, which does not matter if there are free cpus
    	    	if level == 1 and timeok and host.users = 0 then
 		    true -> host.available;
		    quitloop;
   	    	    ;;; check if a console user is on
    	    	elseif level == 2  and timeok and not(host.consoleuser) then
 		    true -> host.available;
		    quitloop;
		    ;;; levels 3 and 4 need to be checked by the client directly
    	    	elseif level == 5  and timeok and (not(host.consoleuser) or host.idleuser) then
 		    true -> host.available;
		    quitloop;
		endif;
	    endfor;
	;;; no criterion, hence host is available if there are available CPUs
	else
	    true -> host.available;
	endif;
    else
	;;; set the time for ping check
        sysdaytime() -> host.lastchecked;
    endif;
    if host.available and verbose then
	npr('Host ' >< host.name >< ' available.');
    elseif verbose then
	npr('Host ' >< host.name >< ' unavailable.');
    endif;
    sysflush(pop_charout_device);
    ;;;if verbose then npr(printhostinfo(host)); endif;
enddefine;

;;; a constructor function returning an host object that updates its information in regular intervals
;;; if CHECKHOSTS is false, then hosts are not checked, but threads are still kept alive to be able to turn
;;; host checking temporarily off
define startcheckinghost(hostinfo) -> host;
    lvars (hname,hOS,hcpusonhost,havailcrit,hsimpath,hpoppath,hscratch) = explode(parselistfromstring(hostinfo)),
          (shname,shOS,shcpusonhost,shavailcrit,shsimpath,shpoppath,sscratch) = explode(listfromstring(hostinfo));
    ;;; create new host object
    lvars host = newsimhost();
    hname ->host.name;
    hOS -> host.OS;
    hcpusonhost -> host.cpusonhost;
    havailcrit -> host.availcrit;
    hsimpath -> host.simpath;
    hpoppath -> host.poppath;
    hscratch -> host.scratch;
    ;;; store the unparsed string ...
    shavailcrit -> host.availcritstr;
    ;;; TODO: get the IP address of the host, so you won't have to do a lookup on the nameserver
    ;;; -> host.IP;
    ;;; create the automatic updating thread, which is started when a host instance is created
    start_thread('t_' >< host.name,
		 10,
		 procedure();
		     sleep_thread(random(100));
		     ;;; keep checking until a message is received, then terminate
		     while not(receive_thread()) do
                 	 ;;; check if hosts need to be checked
		         if CHECKHOSTS then
      		 	     updatehost(host);  ;;; form closure on this host
			 endif;
			 sleep_thread(HOSTCHECKFREQ);  ;;; sleep for a certain time before checking again
		     endwhile;
		 endprocedure,
		 []);
enddefine;


define :method printclients(c:simclient);
    '*** CLIENT INFORMATION ***' >< lfchar ><
    'Cliend ID:           ' >< c.CID >< lfchar ><
	if c.onhost then
	    'On host:             ' >< (c.onhost).name >< lfchar
    else
	    'On host:             ' >< c.onhost >< lfchar
	endif ><
    'Status:              ' >< c.status >< lfchar ><
    'Completedcycles:     ' >< c.completedcycles >< lfchar ><
    'Start time:          ' >< (if c.starttime then sys_convert_date(c.starttime, true) else false endif) >< lfchar ><
    'Last contacted:      ' >< (if c.endtime then sys_convert_date(c.endtime, true) else false endif) >< lfchar ><
    'Simparams:           ' >< c.simparams >< lfchar ><
    'Spop11eval:          ' >< c.spop11eval >< lfchar ><
    'Cpop11eval:          ' >< c.cpop11eval >< lfchar ><
    'Seed:                ' >< c.seed >< lfchar ><
    'Num. interrupts:     ' >< c.numinterruptions >< lfchar ><
    'Num. migrations:     ' >< c.nummigration >< lfchar ><
    'Num. watchdog int.:  ' >< c.numrestarts >< lfchar ><
    'Used hosts:          ' >< c.usedhosts >< lfchar>< lfchar
enddefine;

;;; this returns the command line to start the simulation on a remote host
;;; the following simworld parameters are set directly: checkinterval and imagename
;;; and the ones in the startup parameter list will be ignored
define :method getsimstartcomline(client:simclient);
    lvars host = client.onhost;
    if  sshpath ='' then
    	'cd ' >< host.simpath >< ';' >< host.poppath >< ' :"compile(\'swclient.p\');' ><
    	if (client.experiment).savestate then (client.experiment).savestate else 'false' endif >< '->saveSTATE;' ><
    	'runclient(conslist(\'' >< myhost >< '\',' >< portnumber >< ',\'' >< client.CID >< '\',3));"'
    else
    	;;; create the path, the pop commandline call, compile the client, and start it with the required parameters in a list
    	'cd ' >< host.simpath >< ';' >< host.poppath >< ' :\\\"compile(\'swclient.p\');' ><
    	if (client.experiment).savestate then (client.experiment).savestate else 'false' endif >< '->saveSTATE;' ><
    	'runclient(conslist(\'' >< myhost >< '\',' >< portnumber >< ',\'' >< client.CID >< '\',3));\\\"'
    endif;
enddefine;

define :method getsimcontcomline(client:simclient);
    lvars host = client.onhost;
    ;;; create the path, the pop commandline call, compile the client, and start it with the required parameters in a list, send the CID as an argument
    'cd ' >< host.simpath >< ';' >< host.poppath >< ' +' >< host.scratch >< (client.experiment).uname >< '_' >< client.CID >< '.psv ' >< client.CID
enddefine;


;;; TODO: add the other slots...
define :method printexperiment(ex:experiment);
    lvars item;
    '*** EXPERIMENT INFORMATION ***' >< lfchar ><
    'Name:                ' >< ex.name >< lfchar ><
    'Status:              ' >< ex.status >< lfchar ><
    'Start time:          ' >< ex.starttime >< lfchar ><
    'End time:            ' >< ex.endtime >< lfchar ><
    'Check int. (sec.)    ' >< ex.checkinterval >< lfchar ><
    'Save state:(sec.)    ' >< if ex.savestate then ex.savestate * ex.checkinterval else 'no' endif >< lfchar ><
    'Watch time (sec.)    ' >< ex.watchtime >< lfchar ><
    'Priority:            ' >< ex.priority >< lfchar ><
    'Parallel runs:       ' >< ex.parallelruns >< lfchar ><
    'User:                ' >< ex.uname >< lfchar ><
    'Email:               ' >< ex.email >< lfchar ><
    'Path to results:     ' >< ex.respath >< lfchar ><
    'Path to stats:       ' >< ex.statspath >< lfchar ><
    'Path to images:      ' >< ex.imagepath >< lfchar ><
    'Clients finished:    ' >< ex.finishedclients >< lfchar ><
    'Clients involved in the experiment:' >< lfchar >< lfchar ><
    accumulate(concatenate,'',maplist(ex.clients,printclients)) ><
    '----------------------------------------------' >< lfchar
enddefine;


/*

  [name <string> <respath>]                the name of the experiment and the location where it is supposed to be stored on the server
  [priority <num>]                         a priority from 1 to 9 
  [parallel <num>]                         the number of parallel runs
  [ranseed <num>]                          the ranseed for the experiment
  [notify {NO | <email address>}]          the email address to which to send notification messages
  [savestate {NO | <cycles>}]              whether the state of the client should be saved, frequency in cycles
  [copyimages {NO | <imagepath>}]          the path where simulation states will be stored on the server
  [copystats {NO | <statpath>}]            the path where experiment results will be stored on the server
  [watch {NO | <sec> <sec>]		    restart in sec., regular contact in seconds (roughly)

*/


;;; this parses the parameter list for experiments
;;; either returns true if the parameter list was syntactically correct or an error string
define :method parseserverparams(ex:experiment,parameterlist);
    ;;; default values
    lvars kw,
	 ;;; the list of required keywords for each experiment
	 required = [name parallel user savestate copyimages copystats];

    ;;; default values, no email and lowest priority
    false -> ex.email;
    1 -> ex.priority;
    false -> ex.parallelruns;

    ;;; make sure that all the required keywords are present;
    lvars keywords = getkeys(parameterlist);
    applist(required,procedure(x); unless member(x,keywords) then
 		return('Required keyword missing: ' >< x) endunless;
	endprocedure);

    for kw in parameterlist do
	lvars k = hd(kw);
	if k == "name" then
	    ;;; the name of the experiment
	    kw(2) -> ex.name;
	    kw(3) -> ex.respath;
	elseif k == "priority" then
 	    if not(isnumber(kw(2))) then
		return('Number required for priority: ' >< kw(2));
	    endif;
	    if kw(2) < 1 or kw(2) > 9 then
		return('Priorities must range from 1 to 9: ' >< kw(2));
	    endif;
	    kw(2) -> ex.priority;
	elseif k == "parallel" then
 	    if not(isnumber(kw(2))) then
		return('Number required for parallel runs: ' >< kw(2));
	    endif;
	    ;;; MS: 10-02-03 changed to fix ranseed bug (introduced by the "efficient" client handling...) 	  
	    ;;; if this is true, it must be the ranseed
	    if ex.parallelruns then
		;;; if it is a list then the ranseeds are already in there, ignore for now 
 		unless islist(ex.parallelruns) then
		    [% kw(2), ex.parallelruns %] -> ex.parallelruns;
		endunless;
	    else ;;; just store the number of parallel runs
	    	kw(2) -> ex.parallelruns;
	    endif;
	elseif k == "ranseed" then
	    if not(isnumber(kw(2))) then
		return('Number required for ranseed: ' >< kw(2));
	    endif;
	    ;;; MS: 10-02-03 changed to fix ranseed bug (introduced by the "efficient" client handling...) 
	    ;;;kw(2) -> ranseed;
	    ;;; if this is true, it has been parsed already, so generate the ranseeds for each experiments in a list
            if ex.parallelruns then
		;;; if it is a list then the ranseeds are already in there, ignore for now 
 		unless islist(ex.parallelruns) then
		    ;;; store parallel runs and et the ranseed 
		    [% ex.parallelruns, kw(2) %] -> ex.parallelruns;
		endunless;		
	    else ;;; parallel runs have not been determined yet, so store then ranseed there...
		;;; since the keyword "parallel" is required, this will be filled in eventually
		kw(2) -> ex.parallelruns;
	    endif;
	elseif k == "notify" then
	    
	    kw(2) -> ex.email;
	elseif k == "copystats" then
	    if kw(2) = "NO" then
    		false -> ex.statspath;
	    else  
	    	kw(2) -> ex.statspath;
	    endif;
	elseif k == "copyimages" then
	    if kw(2) = "NO" then
		false -> ex.imagepath;
	    else
	    	kw(2) -> ex.imagepath;
	    endif;
	elseif k == "savestate" then
	    if kw(2) = "NO" then
		false -> ex.savestate;
	    elseif  not(isinteger(kw(2))) then
		return('Integer required for save state: ' >< kw(2))
	    else
	    	kw(2) -> ex.savestate;
	    endif;
	elseif k == "watch" then
	    if kw(2) = "NO" then
		false -> ex.watchtime;
	    elseif  not(isinteger(kw(2))) then
		return('Integer required for watch time: ' >< kw(2));
	    else
	    	kw(2) -> ex.watchtime;
	    endif;
	    if  not(isinteger(kw(3))) then
		return('Integer required for check interval: ' >< kw(3));
	    else
	    	kw(3) -> ex.checkinterval;
	    endif;
	elseif k == "user" then
	    kw(2) -> ex.uname;
	else 
	    unless ignoreUNKNOWNKEYWORD then
 		return('Unkown keyword in parameter list: ' >< k);
	    endunless;
	endif;
    endfor;
    ;;; perform consistency checks on the parameters
    if ex.savestate and ex.checkinterval then
        if ex.savestate < ex.checkinterval then
            return('Value of saving states ' >< ex.savestate >< '  needs to be higher than that of checking ' >< ex.checkinterval);
        else
            ;;; convert the absolute cycle number for save state in a relative cycle number dependent on checkinterval
	    ex.savestate div ex.checkinterval -> ex.savestate;
        endif;
    endif;
    ;;; return false to indicate that there was no error
    false
enddefine;

;;; sets up an experiment with all its clients
;;; parses the list for any list of the form [from x to y step z] and produces subsequent lists with
;;; the respective values substituted
define :method initexperiment(ex:experiment,expinfo)->list_or_errstring;
    lvars list_or_errstring = false;
    ;;; parse the 4th sublist, which is the parameter list for the server
    parseserverparams(ex,expinfo(4)) -> list_or_errstring;
    returnif(list_or_errstring);

    lvars avedims = [];

    ;;; mark the stack so we can see if there are any left over arguments from when we return from parsing...
    popstackmark;
    
    ;;; compute the different simparams lists for the subexperiments and produce clients for them
    define buildparamlist(remlist,avedims);
    	if not(islist(remlist)) then
 	    [%remlist%]
    	elseif null(remlist) then
	    [[]]
    	else
            ;;; concatenate listsofar with each list in what is about to come here
            if hd(remlist) = "from" then
		;;; check syntax first
		unless length(remlist)=6 then
		    true -> list_or_errstring;
		    exitto(initexperiment);
		elseif remlist(3) /= "to" or remlist(5) /= "step" or not(isnumber(remlist(2))) or not(isnumber(remlist(4))) or not(isnumber(remlist(6))) then
		    true -> list_or_errstring;
		    exitto(initexperiment);
		endunless;
            	;;; produce a list
	    	lvars sublist = [% lvars i;for i from remlist(2) by remlist(6) to remlist(4) do i endfor %],
		     restlist = if length(remlist) > 6 then tl(tl(tl(tl(tl(tl(remlist)))))) else [] endif,
		     dimlist = [%allbutlast(length(remlist) - 6,remlist)%];
            	flatmap(sublist,
		    	procedure(startlist);
			    maplist(buildparamlist(restlist,avedims <> dimlist),procedure(x);conspair(startlist,x);endprocedure);
                    	endprocedure);
            else
	    	;;;            lvars tllists = buildparamlist(tl(remlist),namestr);
            	flatmap(buildparamlist(hd(remlist),avedims),
                    	procedure(startlist);
                    	maplist(buildparamlist(tl(remlist),avedims),
				procedure(x);conspair(startlist,x);endprocedure);
                    endprocedure);
   	    endif;
    	endif;
    enddefine;
    
    lvars subexplist = buildparamlist(allbutlast(1,expinfo),avedims);    

    ;;; an empty subexplist indicates an error
    if list_or_errstring or not(null(sysconslist()->>list_or_errstring)) then
	'Error parsing at: ' >< list_or_errstring -> list_or_errstring;
	return;
    endif;

    ;;; check it the path exists, otherwise create the last subdirectory
    unless sys_file_exists(ex.respath) then
        sysobey('/bin/mkdir ' >< ex.respath);
	;;;        16877 -> sysfilemode(ex.respath);
    endunless;


    ;;; produce separate experiments for all subexperiments
    ;;; return the list of experiments
    [% lvars i, newranseed = false; 
       ;;; MS: 10-02-03 changed to fix ranseed bug (introduced by the "efficient" client handling...) 
       ;;; convert the list with parallel runs and ranseed back and use the ranseed now to generate all the seeds for the experiments, 
       ;;; this needs to run without interruptions
       false -> pop_asts_enabled;
       if islist(ex.parallelruns) then
	   (ex.parallelruns)(2) ->> newranseed -> ranseed;
    	   (ex.parallelruns)(1) -> ex.parallelruns;
       endif;

       fast_for i from 1 to length(subexplist) do
	   ;;; create a new experiment
	   lvars params = subexplist(i),
 		;;; produce an exact copy of the experiment
                e = copydata(ex);   ;;;newexperiment();

       	   ;;; MS: 10-02-03 changed to fix ranseed bug (introduced by the "efficient" client handling...) 
	   if newranseed then
	       newranseed -> ranseed;
           endif;
           lvars seeds = [% repeat e.parallelruns times random(100000) endrepeat %];
	   ;;; TODO: use the avedims information to generate names for the experiments...
	   ;;;output and savestates need to be removed, the ranseed needs to be either replaced or added
	   if length(subexplist) = 1 then
	       ex.name -> e.name;
           else
	       ex.name >< '_' >< i -> e.name;
	   endif;
           ;;; remove several entries from the simulation parameters that are produced later
	   removekeyvalue(params(1),"output") -> params(1);
	   removekeyvalue(params(1),"ranseed") -> params(1);	  

           ;;; MS: 06-18-03 added closure for efficiency, this will add procedure that needs to be still executed
           ;;; produce closure and execute only when the experiment is actually scheduled, so clients won't be created until then
	   procedure(p,e1,s);
	       ;;; for each instance produce clients for all parallel runs and add the list to the experiment
	       [% repeat e1.parallelruns times
		      ;;; create a new simclient with a new random seed based on the ranseed, which was set for the experiment
		      lvars client = newsimclient();
		      ;;; save the simparams and add the newly created seed from the client
		      ;;;		      [% conspair([ranseed %client.seed%],p(1)),p(2),p(3)%] -> client.simparams;
           	      ;;; MS: 10-02-03 changed to fix ranseed bug (introduced by the "efficient" client handling...) 
		      ;;; take off the first random seed, then advance list
		      [% conspair([ranseed %hd(s)%],p(1)),p(2),p(3)%] -> client.simparams;
		      tl(s) -> s;
		      e1 -> client.experiment;
		      ;;; if an imagepath is provided create a client specific name for it
		      if e1.imagepath then
			  e1.imagepath >< e1.uname >< '_' >< client.CID >< '.psv' -> client.theimage;
		      endif;
		      client
		  endrepeat %]
	   endprocedure(%params,e,seeds%) -> e.clients;
	   
	   ;;; finally add experiment to list and return experiment list
	   e
       endfast_for;
       ;;; enable interrupts again
       true -> pop_asts_enabled %] -> list_or_errstring;
enddefine;

;;; returns true if all clients are done
define :method allclientsfinished(ex:experiment);
    dlocal pop_asts_enabled = false;

    lvars item;
    fast_for item in ex.clients do
;;;	npr(item.CID >< ' ' >< item.status);
	returnif(item.status /= "FINISHED")(false);
    endfast_for;
    true
enddefine;

;;; loads and returns an experiment from file using datafile
define :method loadexperiment(es:experimentserver,expfilepath) -> ex;
    datafile(expfilepath) -> ex;
enddefine;


;;; this will load the host file
define :method loadhosts(es:experimentserver,hostsfilepath);
    lvars i, hrpt, hostinfo;
    line_repeater(hostsfilepath,255) -> hrpt;
    while ((hrpt()->>hostinfo) /= termin) do
	;;; extract parts from the line: name of host,OS of host,# of cpus on host,the checkMIGRATION list
	;;; store it in hostlist and start checking the host
;;;	npr('host: ' >< hostinfo);
;;;	explode(listfromstring(hostinfo)) =>
     	conspair(startcheckinghost(hostinfo),es.hostlist) -> es.hostlist;
    endwhile;
    ;;; Host threads are started, now start garbage collection thread
/*
    start_thread('t_garbage',
		 10,
		 procedure();
    		     dlocal pop_asts_enabled = false;
		     while true do
			 sysgarbage();
			 ;;;true -> pop_asts_enabled;
			 sleep_thread(HOSTCHECKFREQ);
			 ;;;false -> pop_asts_enabled;
		     endwhile;
		 endprocedure,
		 []);
*/
enddefine;

;;; MS 06-18-03: added priority mechanisms to allow migrating or restarting clients to obtain hosts before scheduled clients
;;; this implements a blocking call: will return only if a host is available (which then will be returned)
;;; NOTE: a host that is returned here is automatically marked as used, so whoever uses it should free it if it is not used anymore
define :method getavailablehost(es:experimentserver,old);
    dlocal pop_asts_enabled = false;

    lvars host;    
    ;;; a client requires a host, if it is an old one, put it the priority list
    if old then
        pnumclientswaiting + 1 -> pnumclientswaiting;
    else
        numclientswaiting + 1 -> numclientswaiting;
    endif;	
    ;;; start checking hosts, if host checking is required
    if mustCHECKHOSTS then
        true -> CHECKHOSTS;
    endif;
    repeat
	;;; disable thread switching, use this for mutual exclusion...  (should be synchronized in JAVA)

       	fast_for host in es.hostlist do
            ;;; skip no priority clients if clients with priority are waiting too
            if (pnumclientswaiting > 0 and old) or (pnumclientswaiting = 0) then
    	        ;;; if host has been checked at least once and is marked available, it should be available
	        if host.lastchecked and host.available then
		    ;;; decrease the number of available cpus on the host....
	    	    host.cpusused + 1 -> host.cpusused;
	    	    if host.cpusused >= host.cpusonhost then
		       false -> host.available;
	    	    endif;
		    ;;; reduce the number of waiting clients
		    if old then
		        pnumclientswaiting - 1 -> pnumclientswaiting;
                    else
 		        numclientswaiting - 1 -> numclientswaiting;
                    endif;
		    if numclientswaiting = 0 and pnumclientswaiting = 0 then
		        ;;; stop checking host, there are available ones
                        if mustCHECKHOSTS then
		            false -> CHECKHOSTS;
                        endif;
		    endif;
		    ;;; enable interrupts
		    ;;;true -> pop_asts_enabled;
		    return(host);
                endif;
	    endif;
	endfast_for;
	;;; enable thread switching
	;;;true -> pop_asts_enabled;
	if verbose then npr('No hosts available at present, waiting for ' >< WAITINGFORHOSTS >< ' sec. \n'); endif;
	;;; wait for the appropriate time
	sleep_thread(WAITINGFORHOSTS);
    endrepeat;
enddefine;


define :method copyimagefromclient(client:simclient);
    dlocal interrupt = identfn, pop_file_versions = false, pop_status;
    lvars hostimage = (client.onhost).scratch >< (client.experiment).uname >< '_' >< client.CID >< '.psv',
	 cimage = client.theimage;
   
    ;;; if an image exists, change its filename temporarily to make sure we won't screw it up during transfer it an error occurs
    if sys_file_exists(cimage) then
    	;;; delete it if it exists    
	if verbose then npr('Renaming old image'); endif;
	sys_file_move(cimage,cimage >< '.old');
    endif;

    if verbose then npr('Copying image for client ' >< client.CID >< ' to server from remote host ' >< (client.onhost).name); endif;
    ;;; produce a verification filename
    lvars ret,verify='; touch ' >< cimage >< '.chk';
    if  sshpath /='' then
    	true -> client.imagebeingcopied;
	sysobeynew(scppath >< (client.experiment).uname >< '@' >< (client.onhost).name >< ':' >< hostimage >< ' ' >< cimage >< verify,
                   SCPTIMEOUT,cimage >< '.chk',true)->ret;
    	false -> client.imagebeingcopied;	
    else
    	sysobey('/bin/cp ' >< hostimage >< ' ' >< (client.experiment).imagepath);
    	(pop_status = 0) -> ret;
    endif;

    ;;;if client.status = "MIGRATING" then npr('*** IMAGE copied for client ' >< client.CID); endif;
    ;;; check if the transfer was successful: ret has the exit status of the child if successful
    if ret then
	;;; remove the security copy if there was one
	if sys_file_exists(cimage >< '.old') then
	    if deleteit(cimage >< '.old') and verbose then npr('Could not delete old image file'); endif;
	endif;
    else ;;; there was a problem, so use the old version
 	if sys_file_exists(cimage >< '.old') then
	    if verbose then npr('Problem with image transfer for client ' >< client.CID >< ' -- old image is the best we got...'); endif;
	    sys_file_move(cimage >< '.old',cimage);
	endif;
    endif;
enddefine;


;;; MS: 03-17 added mutual exclusion
define :method copyimagetoclient(client:simclient)/* -> success */;
    dlocal interrupt = identfn;
    ;;; make sure nobody deletes the client data structure in the meantime...
    dlocal pop_asts_enabled = false, pop_status;
      
    ;;; check if clientID is still good, if not return, also return if the experiment is already done... 
    returnif(isstring(client.experiment) or not(client.onhost))(false);
    lvars scpcmd = scppath >< client.theimage >< ' ' >< (client.experiment).uname >< '@' >< (client.onhost).name >< ':' >< (client.onhost).scratch;
    ;;; now enable interrupts again
    true -> pop_asts_enabled;

    if verbose then npr('Copying image for client ' >< client.CID >< ' to remote host ' >< (client.onhost).name); endif;
    ;;; now copy the image from the server to the remote host
    if  sshpath /='' then
	lvars verify=';touch ' >< client.theimage >< '.chk';
    	sysobeynew(scpcmd><verify,SCPTIMEOUT,client.theimage><'.chk',true);
    else
    	sysobey('/bin/cp ' >< client.theimage >< ' ' >< (client.onhost).scratch);
    	(pop_status = 0);
    endif;
enddefine;

define :method copyfilefromclient(client:simclient,pathonclient,pathonserver)/* -> success */;
    dlocal interrupt = identfn;

    ;;; now copy the latest image
    if  sshpath /='' then
	lvars verify=';touch ' >< pathonserver >< '.chk';
    	sysobeynew(scppath >< (client.experiment).uname >< '@' >< (client.onhost).name >< ':' >< pathonclient >< ' ' >< pathonserver><verify,
                   SCPTIMEOUT,pathonserver><'.chk',true);
    else
    	sysobey('/bin/cp ' >< pathonclient >< ' ' >< pathonserver);
    	(pop_status = 0);
    endif;
enddefine;


define :method copyfiletoclient(client:simclient,pathonclient,pathonserver)/* -> success */;
    dlocal interrupt = identfn;    
    ;;; now copy the latest image
    if  sshpath /='' then
	lvars verify=';touch ' >< pathonserver >< '.chk';
    	sysobeynew(scppath >< pathonserver >< ' ' >< (client.experiment).uname >< '@' >< (client.onhost).name >< ':' >< pathonclient><verify,
                   SCPTIMEOUT,pathonserver><'.chk',true);
    else
    	sysobey('/bin/cp ' >< pathonserver >< ' ' >< pathonclient);
    	(pop_status = 0);
    endif;
enddefine;


;;; continues a client for which a saved image exists (NOT in a separate thread as this will be called from a thread already, e.g., serveclient)
define :method continueclient(client:simclient);
    dlocal interrupt = identfn;
    lvars newhost = client.onhost;
    ;;; TODO: should this be synchronized???
    ;;; store the new host in the client's host history list
    unless member(newhost.name,client.usedhosts) then
    	conspair(newhost.name,client.usedhosts) -> client.usedhosts;    
    endunless;
    unless member(client.CID,newhost.clients) then
    	conspair(client.CID,newhost.clients) -> newhost.clients;
    endunless;

    ;;; put the new availcrit into cpop11eval and send it to the simulation
    '\'' >< newhost.scratch >< (client.experiment).uname >< '_' >< client.CID >< '.stats\'->saveSTATSFILE;' 
       >< '\'' >< newhost.scratch >< (client.experiment).uname >< '_' >< client.CID >< '.psv\'->imagename;'
       >< newhost.availcrit >< '->checkMIGRATION;' -> client.cpop11eval;
       
    if sshpath = '' then
    	sysobey(getsimcontcomline(client));
    else
    	lvars sret = sysobeynew(sshpath >< newhost.name >< ' "' >< getsimcontcomline(client) >< '"', SSHTIMEOUT);
    endif;
    if verbose then npr('Client ' >< client.CID >< ' restarted on host ' >< newhost.name); endif;
enddefine;

;;; keeps trying to get a new host and transfers image there until successful
define :method transferimage(es:experimentserver,client:simclient);
    ;;;npr('IN TRANSFER IMAGE for client ' >< client.CID);
    getavailablehost(es,true) -> client.onhost;
    ;;; if you can't copy images to the client, try new hosts...
    until copyimagetoclient(client) do
    	;;;npr('IN TRANSFER IMAGE: COPY FAILED TRYING TO COPY for client ' >< client.CID);
	;;; mark host as unavailable
	false -> (client.onhost).available;	
	;;; wake up the thread associated with the client so it can update the host information
	wakeup_thread('t_' >< (client.onhost).name);
	
	;;; mark client without host and get a new host (this will block until a host is available)
	false -> client.onhost;
	getavailablehost(es,true) -> client.onhost;
    enduntil;			
enddefine; 

define removefileonhost(hostname,filepathonhost)/* -> success */;
    dlocal interrupt = identfn;    
    if verbose then npr('Removing ' >< filepathonhost >< ' on ' >< hostname); endif;
    sysobeynew(sshpath >< hostname >< ' "/bin/rm ' >< filepathonhost >< '"',SSHTIMEOUT);
    if verbose then npr('Removed ' >< filepathonhost >< ' on ' >< hostname); endif;
enddefine;

;;; starts a client in a separate thread and returns
define :method startclient(es:experimentserver,cname,cobject,host,prior);
    dlocal interrupt = identfn, pop_asts_enabled;
    ;;; just make sure the thread does not exist, otherwise wait for it to finish
    while exists_thread(cname) do
	;;; kill the thread and sleep for only a short while, that should be sufficient for the other thread to die
	;;; MS: 04/23/  ADDED kill_thread
	kill_thread(cname);
	sleep_thread(1);
	npr('Thread ' >< cname >< ' exists, waiting at ' >< sysdaytime());
    endwhile;
    ;;;npr('cname: ' >< cname);
    ;;; start a thread that will start the remote simulation client
    start_thread(cname,
		 prior,
		 procedure(client,h);
		     ;;; mark client as starting and add it to the client list in the host
		     ;;; and add the host to the list of hosts used by this client
		     conspair(h.name,client.usedhosts) -> client.usedhosts;
                     ;;; SHOULD WE ADD THE HOST OBJECT HERE?
		     h -> client.onhost;
		     conspair(client.CID,h.clients) -> h.clients;
		     ;;; also add the host availability criterion to the start evaluation string
		     ;;; to set the checkMIGRATION global in the client, will be sent together with the simparams
		     h.availcritstr >< '->checkMIGRATION;' -> client.spop11eval;
		     if verbose then npr('Starting client ' >< client.CID >< ' on host ' >< h.name); endif;
		     ;;; set the start time of the client, this is used to check whether it actually started...
	    	     sys_real_time() -> client.starttime;
		     "INITIALIZING" -> client.status;
		     ;;; this will return right away
		     if sshpath = '' then
    		     	 sysobey(getsimstartcomline(client));
		     else
			 lvars sret = sysobeynew(sshpath >< h.name >< ' "' >< getsimstartcomline(client) >< '"', SSHTIMEOUT); ;;;, '> ' >< outfile %]);
		     endif;
		     ;;; check if the status of the client is still "INITIALIZING", if so mark it for forced "WATCHDOGRESTART":
                     ;;; if in the meantime the client should switch to starting, the better!
		     if client.status = "INITIALIZING" then
			 false -> pop_asts_enabled;
			 "WATCHDOGRESTART" -> client.status;
			 true -> pop_asts_enabled;
                     endif;
		     if verbose then npr('Client ' >< client.CID >< ' started at ' >< (if client.starttime then sys_convert_date(client.starttime, true) else false endif)); endif;
		 endprocedure,
		 [% cobject, host %]);
enddefine;


;;; will serve a client connected through comm_sock
;;; the message format used is a list of three elements: clientID, message type and (optional) data
;;; the following message types are coded:
;;;   PREQ       - request simulation parameters
;;;   SINT data  - simulation interrupted, image saved in imagefile, data contains the level that caused the interruption
;;;   SCON       - simulation ready to continue, image loaded, waiting for continuation parameters
;;;   SPAR data  - the simulation parameters
;;;   CPAR data  - the string to be evaluated before the simulation continues
;;;   DONE       - the client is done, the server can get the statsfile
;;;   SSAV data  - notification the simstate was saved, contains the current cycle
;;;   CCYC data  - contains the current sim cycle
define :method serveclient(es:experimentserver,c_sock);
    dlocal interrupt = identfn;
    dlocal pop_asts_enabled, pop_file_versions = false;

    ;;; keep serving as long as needed
    lvars message, ack, client = false, ex = false;
    ;;;  if verbose then npr('Client connected...'); endif;
    ;;; try to keep the socket open with the client
    repeat
	;;; need to check here whether socket was closed because we are done, or because there was an error, in either case return
	unless (receive_message(SID,c_sock)->>message) then
	    ;;; MS: ----------------------------------------- ERROR HANDLING ---------------------------------------
	    ;;; there may have been an error
	    ;;; take the last client and check its state
	    if client then
		if verbose then npr('Communication with client ' >< client.CID >< ' interrupted!'); endif;
	    else 
		;;; otherwise this was the first time around, either an unauthorized client connected, or there was some sort of error, so just quit
		if verbose then npr('Error on socket'); endif;
	    endif;
	    return;		
	endunless;
	;;; get the message details and the client information
   	lvars (mID, mtype, mdata) = explode(message);
	;;; MS: 02-05-03 make sure MID is a string, could have been a word if the above 
        ;;; unless clause is true
	(es.allclients)(consword(mID >< '')) -> client;
	;;; just to be sure, if this is false, the client is not known to the experiment server, so just discard it 
	returnunless(client);
	;;; and mark client's last contact time
	sys_real_time() -> client.endtime;
	;;; the client was in the experiment server and we got the record
	;;; set the experiment of the client
	client.experiment -> ex;
	;;; update the socket used by the client, just in case, this should not be necessary
	c_sock -> client.comm_sock;     
	;;; now check and make sure the CID is still in the experiment server, otherwise the watchdog must have removed it, so quit
	returnunless((es.allclients)(consword(mID >< '')));
	;;; the client still exists, so check the message type
	if verbose then npr('Received ' >< mtype >< ' from client ' >< mID); endif;
	;;;sysflush(pop_charout_device);
	if mtype == "SSAV" or mtype == "CCYC" then
	    if mtype == "SSAV" then
		;;; save the current cycle information
	    	client.completedcycles + ex.checkinterval -> client.completedcycles;
		;;;if verbose then npr('Client ' >< mID >< ' saved state at cycle ' >< mdata); endif;
	    	;;; if images are supposed to be copied, get the most recent one
	    	if ex.imagepath then
		    if client.imagebeingcopied then
			;;; kill the copying thread as it is too slow
			kill_thread(client.CID >< 'copyimage');
			;;;npr('Killing copier as it is too slow...');
		    endif;
		    ;;; wait for the old copier to finish
		    while exists_thread(client.CID >< 'copyimage') do
			;;; sleep for only a short while, that should be sufficient
			sleep_thread(1);
		    endwhile;
		    ;;; start a new copier
		    start_thread(client.CID >< 'copyimage',100,copyimagefromclient,[% client %]);
	    	endif;
		;;; if we are still in "STARTING", change to "RUNNING" as the image has been saved locally
		if client.status = "STARTING" then
		    "RUNNING" -> client.status;
		endif;
	    else
		;;; save the current cycle information
	    	mdata -> client.completedcycles;
	    endif;
	    ;;; keep connection alive
	elseif mtype == "PREQ" or mtype == "SCONT" then
            ;;; send the respective startup or continutation parameters, quit if the socket got closed
	    if mtype == "PREQ" then
		;;; add to simparams the information about the scratch space on the host
		lvars spa = client.simparams;
		;;;		client.simparams==>
		if ex.checkinterval then
		    ;;; remove the old savestate line and put in the new
 		    removekeyvalue(spa(1),"savestate") -> spa(1);
		    [% conspair([savestate %ex.checkinterval% %(client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.psv'%],spa(1)),spa(2),spa(3)%] -> spa;
		endif;
 		removekeyvalue(spa(1),"statsfile") -> spa(1);
		[% conspair([statsfile %(client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.stats'%],spa(1)),spa(2),spa(3)%] -> client.simparams;
            	;;; construct a pair of additional startup parameters
	    	returnunless(send_message(SID,"SPAR",conspair(client.spop11eval,client.simparams),c_sock)->>ack);
	    else
		returnunless(send_message(SID,"CPAR",client.cpop11eval,c_sock)->>ack);
	    endif;
	    ;;; check for error packet
	    if islist(ack) then
		npr('ERROR in communication');
		;;; simple error protocol: the message was "EWFM", so close socket and wait for client to start over
		return;
	    endif;
	    ;;; otherwise update the client state	
	    if mtype == "PREQ" then
		;;; this is to indicate that no image of the client is available (used in watch dog to decide whether to restart or continue the client)
		;;; MS: 04/23: leave status as STARTING if images are to be saved, RUNNING otherwise
		if client.theimage then
	    	    "STARTING" -> client.status;
		else
    		    "RUNNING" -> client.status;
		endif;
	    	if verbose then npr('Initial parameters sent, simulation is running on host ' >< (client.onhost).name); endif;
	    else
		;;; MS: 04/23: leave status as STARTING if no image is available, RUNNING otherwise
		if client.theimage then
		    if sys_file_exists(client.theimage) then
			"RUNNING" -> client.status;
		    else
			"STARTING" -> client.status;
		    endif;
		else
		    "RUNNING" -> client.status;
		endif;
	    	if verbose then npr('Continuation parameters sent, simulation is continuing on host ' >< (client.onhost).name); endif;
	    endif;
	    if exists_thread(client.CID) then
		kill_thread(client.CID);
	    endif;
	    ;;; keep connection alive
	elseif mtype == "SINT" then
	    ;;; check the level of interruption: migrate for any level /= 3
	    ;;; close and reset the socket, so it can be garbage collected...
	    sysclose(c_sock);
	    [] ->> c_sock -> client.comm_sock;
	    lvars newhost = client.onhost;		
	    
	    ;;; check if we need to migrate
	    if mdata /= 3 then
		"MIGRATING" -> client.status;
		client.nummigration + 1 -> client.nummigration;
		;;;npr('*** CLIENT ' >< client.CID >< ' is migrating');

		;;; mark host as unavailable and erase all clients on it
		false -> (client.onhost).available;
		[] -> (client.onhost).clients;
	    	;;; wake up the thread associated with the client so it can update the host information
		wakeup_thread('t_' >< (client.onhost).name);

		unless client.theimage then
		    testhostfilescratch dir_>< (client.experiment).uname >< '_' >< client.CID >< '.psv' -> client.theimage;
		    testhostfilescratch >< '/' -> ex.imagepath;
		endunless;
		;;; copy the image from the host to the local scratch space		
		copyimagefromclient(client);
		;;; if we don't have an image from the client, it needs to be started from scratch
		unless sys_file_exists(client.theimage) then
		    if verbose then npr('IMAGE DOES NOT EXIST, YET CLIENT IS MIGRATING, STARTING ANEW...'); endif;
		    getavailablehost(es,true) -> client.onhost;
		    "SCHEDULED" -> client.status;
		    startclient(es,client.CID,client,client.onhost,100);
		    return;
		;;; image existed, clean up the old one in the scratch on the host
		else
		    if removefileonhost((client.onhost).name,(client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.psv') then
		    ;;;npr('PSV removed from remote host');
		    else
			if verbose then npr('PSV was NOT removed from remote host'); endif;		
		    endif;
		endunless;

		;;; mark client without host and get a new host (this will block until a host is available)
		false -> client.onhost;

		;;; if copying is required, we have to make sure that the image will actually get copied, otherwise just get a host
	    	if ex.imagepath then
		    transferimage(es,client);
		else
		    getavailablehost(es,true) -> client.onhost;
		endif;
		;;; now that a host is available, mark it as restarting
		"RESTARTING" -> client.status;
		npr('SINT: Client will be restarted on host ' >< client.onhost);
	    else
		npr('SINT: Client can continue on host ' >< client.onhost);
		client.numinterruptions + 1 -> client.numinterruptions;
	        ;;; host can be reused, simply start the simulation again, no image would need to be transferred, but:
	        ;;; if images are available, we might as well get the most recent one
 	    	if ex.imagepath then
		    if client.imagebeingcopied then
			;;; kill the copying thread as it is too slow
			kill_thread(client.CID >< 'copyimage');
			;;;npr('Killing copier as it is too slow...');
		    endif;
		    ;;; wait for the old copier to finish
		    while exists_thread(client.CID >< 'copyimage') do
			;;; sleep for only a short while, that should be sufficient
			sleep_thread(1);
		    endwhile;
		    ;;; start a new copier
		    start_thread(client.CID >< 'copyimage',100,copyimagefromclient,[% client %]);
	    	endif;
		"RESTARTING" -> client.status;
	    endif;
	    continueclient(client);
	    ;;; quit as the client will contact the server anew
	    return;
	elseif mtype == "DONE" then
	    ;;; close the socket just in case
	    sysclose(c_sock);
	    [] ->> c_sock -> client.comm_sock;
	    ;;; try to get the stat file, if it does not work after three attempts
	    if ex.statspath then
	    	lvars attempts = 3;
	    	until attempts = 0 or copyfilefromclient(client,
							 (client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.stats',
							 ex.statspath >< ex.name >< '_' >< client.CID >< '.stats') do
               	    attempts - 1 -> attempts;
	    	enduntil;
	    	;;; if this failed then simply quit, the watchdog will try to restart this
		;;; MS: 04/23 IF THE CLIENT FAILED TO SAVE THE STATS, THEN THIS WILL FAIL TOO, AS THE FILE WON'T BE THERE...
                ;;; the watchdog should take care of it
		if attempts = 0 then 
		    npr('STATS NOT COPIED, watchdog will restart this eventually');
                    ;;; so simply restart the client		    
		    return;
		else
	    	    if verbose then npr('STATS COPIED!'); endif;		
		    if removefileonhost((client.onhost).name,(client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.stats') then
			if verbose then npr('STATS removed from remote host'); endif;
		    else
	    	    	if verbose then npr('STATS were NOT removed from remote host'); endif;		
		    endif;
		endif;
	    endif;
	    ;;; the stats are here so mark the client as finished
	    false -> pop_asts_enabled;
	    ;;; save the path to the stats file in the client's experiment slot instead of the reference to the experiment data structure
	    ex.statspath >< ex.name >< '_' >< client.CID >< '.stats' -> client.experiment;
	    ;;; since client finished, check whether the experiment is complete and whether all experiments finished
	    "FINISHED" -> client.status;
	    ex.finishedclients + 1 -> ex.finishedclients;
	    ;;; remove client from host structure
	    ncdelete(client.CID,(client.onhost).clients) -> (client.onhost).clients;
	    ;;; remove any old images on the server if there are any
	    true -> pop_asts_enabled;
	    ;;; if there is an image, clean up the server scratch space
	    if client.theimage then
 	        ;;; remove the old one on the host
	        if removefileonhost((client.onhost).name,(client.onhost).scratch >< ex.uname >< '_' >< client.CID >< '.psv') then
		        ;;;npr('PSV removed from remote host');
		else
	    	    	    if verbose then npr('PSV was NOT removed from remote host'); endif;		
		endif;
		if verbose then npr('Removing old images on server for client ' >< client.CID); endif;
	    	if sys_file_exists(client.theimage) then
 	            if deleteit(client.theimage) and verbose then npr('Could not delete the image file'); endif;
	    	endif;
 	    	if sys_file_exists(client.theimage >< '.old') then
	            if deleteit(client.theimage >< '.old') and verbose then npr('Could not delete the old image file'); endif;
	    	endif;
	    endif;
	    ;;; remove the host from the client 
	    false -> client.onhost;  ;;; mark client without host
	    ;;; disable thread switching, use this for mutual exclusion...  (should be synchronized in JAVA)
	    false -> pop_asts_enabled;
	    if allclientsfinished(ex) and (ex.status /= "FINISHED") then
    		"FINISHED" -> ex.status;
		;;; set the end time of the experiment
		sysdaytime() -> ex.endtime;
		;;; need to do this before we save the experiment...
		ncdelete(ex,es.runningexps) -> es.runningexps;
		;;; store the name and location of the finished experiment
		ex.respath  -> (es.finishedexps)(ex.name);
		;;; remove all clients of the experiment from the experiment server to prevent memory overflow as they are not 
		;;; needed any longer				
		fast_for client in ex.clients do
		    false -> (es.allclients)(client.CID);
		    ;;; TODO: rename the clients to get consecutive ordering...
		    ;;; rename the 
		endfast_for;
                ;;; save the experiment
		ex -> datafile(ex.respath dir_>< ex.name >< '.exp');
		true -> pop_asts_enabled;
    		if verbose then npr('Experiment ' >< ex.name >< ' finished and saved...'); endif;		
		;;; notify user by email, if required
    		if ex.email then
		    pipeout(stringin('Experiment ' >< ex.name >< 'finished'),
			    '/bin/mail',[%'/bin/mail', '-t', ex.email%], false);
    		endif;
		;;; if this was the last running experiment, let experiment server know
		if null(es.runningexps) then
		    true -> es.allfinished;
	            if verbose then npr('All experiments finished and saved...'); endif;
         	    ;;; if not in interactive mode, the server needs to shut down, so
		    ;;; kill all update threads to stop the thread package and end the server
		    unless interactiveMODE then
		     	kill_all_threads();
		    endunless;
    		endif;
	    else
		;;; enable interrupts again
		true -> pop_asts_enabled;
	    endif;
	    return;
	else
 	    if verbose then npr('Unknown packet'); endif;
	    return;
	endif;
    endrepeat;
enddefine;

;;; returns the next runnable experiment according to priority or false if all experiments have been completed
;;; and removes it from the queue
define :method nextrunnableexperiment(es:experimentserver);
    dlocal pop_asts_enabled = false;
    lvars i,exs,ex;
    for i from 9 by -1 to 1 do
	unless null((es.experiments)(i) ->> exs) then
	    ;;; take the experiment off the queue and move it to running experiments
	    hd(exs) ->> ex;
	    es.runningexps <> [% ex %] -> es.runningexps;
	    tl(exs) -> (es.experiments)(i);
	    return(ex);	
	endunless;
    endfor;
    false
enddefine;

;;; returns a list of pairs of priority and name of all queued experiments, ordered by priority
define :method queuedexperiments(es:experimentserver);
    dlocal pop_asts_enabled = false;
    lvars i,exs;
    [% for i from 9 by -1 to 1 do
	   unless null((es.experiments)(i) ->> exs) then
	       applist(exs,procedure(x);[% x.name, i%] endprocedure)
	   endunless;
       endfor %]
enddefine;


;;; this is the watch dog that makes sure that clients make enough progress over time, otherwise they will be restarted
define :method clientwatchdog(es:experimentserver);
    repeat
	false-> watchdogRUNNING;
	;;; check in regular intervals that all clients have report back and made progress
	sleep_thread(WATCHDOGINTERVAL);
	true-> watchdogRUNNING;
	;;; perform checks for all clients
	;;;lvars (ehour,emin,esec) = sys_parse_string(substring(12,8,sysdaytime()),`:`,strnumber);
	lvars esec = sys_real_time();
	;;; if verbose then npr('WATCHDOG: waking up at: ' >< sysdaytime()); sysflush(pop_charout_device); endif;
	npr('WATCHDOG: waking up: ' >< sysdaytime()); sysflush(pop_charout_device);
	;;; this makes a temporary copy of the property, so it won't be affected by adding or deleting clients, this is important so that new clients won't be examined the first time around 
	appproperty(allclients(es),
		    procedure(clientID,client);
    			dlocal pop_asts_enabled = false;
			;;; MS: 03-17 add mutual exclusion here to avoid race condition with transferring image
			;;;false -> pop_asts_enabled;
			if verbose then npr('WATCHDOG: checking client: '><clientID); sysflush(pop_charout_device); endif;
			;;; first check if the client should be watched
			lvars ex = client.experiment;
			;;; check if the experiment has been removed already, then it is a string, so ignore it, also ignore it if there is no watchtime set
			if isstring(ex) or not(ex.watchtime) then
			    ;;;true -> pop_asts_enabled;
			    return;
			endif;
			;;; make sure the client is actually running (and not already finished, or in the process of migrating, or even only scheduled...) 
			;;; check if the client has at least initalized, otherwise status = false
			if client.status and client.status /= "FINISHED" and client.status /= "SCHEDULED" then
			    if verbose then npr('WATCHDOG: ' >< clientID >< ' is ' >< client.status); sysflush(pop_charout_device); endif;
			    ;;; get the last time the client reported back, which is stored in clien.endtime unless the client has just been initialized, then take starttime instead
			    unless client.starttime then
				if verbose then npr('WATCHDOG: ' >< clientID >< ' has false starttime'); sysflush(pop_charout_device); endif;
				npr('WATCHDOG: ' >< clientID >< ' has false starttime, returning...'); sysflush(pop_charout_device);
				return;
				/*
				npr('WATCHDOG: client start time is false for client ' >< clientID);
				npr('Status: ' >< client.status);
				'<experiment>' -> client.experiment;
				client =>
				ex -> client.experiment;
				mishap('Please report this error to mscheutz@nd.edu');
				*/
			    endunless;
			    ;;;lvars (sh,sm,ss) = sys_parse_string(substring(12,8,if client.endtime then client.endtime else client.starttime endif),`:`,strnumber);
			    if verbose then npr('WATCHDOG: Last contact: ' >< (if client.starttime then sys_convert_date(client.starttime, true) else false endif) >< ' ' >< (if client.endtime then sys_convert_date(client.endtime, true) else false endif)); sysflush(pop_charout_device); endif;
			    ;;;lvars lastcontact = sh*3600 + sm*60 + ss + (if client.status = "MIGRATING" then MAXMIGRATIONTIME else ex.watchtime endif);
			    lvars lastcontact = (if client.endtime then client.endtime else client.starttime endif) + (if client.status = "MIGRATING" then MAXMIGRATIONTIME else ex.watchtime endif);
			    ;;; MS: 03-28: check if lastcontact would be after midnight, then subtract 24 hrs.
                            ;;;if lastcontact >= 169200 then lastcontact - 169200 -> lastcontact endif;
			    ;;; if client has not reported back in time, take action			    
		 	    if lastcontact < esec then
				;;; kill the serveclient thread for the client....				
			    	if verbose then npr('WATCHDOG: ' >< clientID >< ' has not reported back in time...'); sysflush(pop_charout_device); endif;
				;;; create a new ID, put it in the client and remove the old one from the server
				;;; this is necessary to prevent clients that contrary to expectation report back after all to mess up the simulation
				false -> (es.allclients)(clientID);
				;;; delete client from client list on host (only if it is on a host)
				if client.onhost then
				    if member(client.CID,(client.onhost).clients) then
					ncdelete(client.CID,(client.onhost).clients) -> (client.onhost).clients;
				    endif;
				endif;
				if exists_thread(client.CID >< 'watchdog') then
				    kill_thread(client.CID >< 'watchdog');
				endif;
				;;; create new client ID
				gensym("cl") -> client.CID;
				;;; unmark the host, the starttime and endtime, and reset the status to scheduling
				false -> client.onhost;
				false -> client.starttime;
				false -> client.endtime;
				;;; put the client back into the property of the experimentserver
				client -> (es.allclients)(client.CID);
				client.numrestarts + 1 -> client.numrestarts;

				;;; check if there is an old image, i.e., if client.status /= INITIALIZING and /= STARTING 
				if client.status /= "INITIALIZING" and client.status /= "STARTING" and client.status /= "WATCHDOGRESTART" and
                                   (if client.theimage then sys_file_exists(client.theimage) else true endif) then
				    "MIGRATING" -> client.status;
				    ;;; get the image and start the client again in a thread
				    if verbose then npr('WATCHDOG: Client ' >< clientID >< ' needs to be continued on a new host'); sysflush(pop_charout_device);endif;
 				    start_thread(client.CID >< 'watchdog',100,
		 				 procedure(es,c);
						     ;;; if copying is required, we have to make sure that the image will actually get copied, otherwise just get a host
	    					     if client.theimage then
						     	 ;;; rename the image to reflect the new CID							 
						     	 if verbose then npr('WATCHDOG: Renaming image for old client ' >< clientID >< '...'); sysflush(pop_charout_device); endif;
							 lvars theoldimage = client.theimage;
                                                         ex.imagepath >< ex.uname >< '_' >< client.CID >< '.psv' -> client.theimage;
	    					     	 sys_file_move(theoldimage,client.theimage); 
		    					 transferimage(es,c);
						     else
		    					 getavailablehost(es,true) -> c.onhost;
						     endif;
						     ;;; we got a host, so mark the client as restarting
						     sys_real_time() -> client.starttime;
				    		     "RESTARTING" -> client.status;
						     npr('WATCHDOG: Client will be restarted on host ' >< c.onhost); sysflush(pop_charout_device);
						     continueclient(c);
						 endprocedure,
		 				 [% es,client %]);
			        else ;;; there is no image, so we need to start from scratch
				    "SCHEDULED" -> client.status;
				    0 -> client.completedcycles;
				    ;;; start this in thread to get an available host, so the watch dog is not blocked in case no host is available at the moment
				    if verbose then npr('WATCHDOG: Client ' >< clientID >< ' needs to be started anew on a new host'); sysflush(pop_charout_device);endif;
				    start_thread(client.CID >< 'watchdog',100,
		 				 procedure(es,c);
				    		     ;;; start the client in a separate thread--a thread by this name should not exist anymore--CHECK
				    		     startclient(es,c.CID,c,getavailablehost(es,true),100);
		 				 endprocedure,
		 				 [% es,client %]);				    
				endif;
			    ;;; check if the client is marked for restart, then restart it with priority
			    elseif client.status = "WATCHDOGRESTART" then
				;;; force a restart of the client
				"SCHEDULED" -> client.status;
				0 -> client.completedcycles;
				;;; start this in thread to get an available host, so the watch dog is not blocked in case no host is available at the moment
				if verbose then npr('WATCHDOG: Client ' >< clientID >< ' needs to be started anew on a new host'); sysflush(pop_charout_device);endif;
				start_thread(client.CID >< 'watchdog',100,
		 			     procedure(es,c);
				    		 ;;; start the client in a separate thread--a thread by this name should not exist anymore--CHECK
				    		 startclient(es,c.CID,c,getavailablehost(es,true),100);
		 			     endprocedure,
		 			     [% es,client %]);
			    else
				;;; this client is OK, so enable interrupts again
			    	;;;true -> pop_asts_enabled;
				if verbose then npr('WATCHDOG: Client ' >< client.CID >< ' OK'); sysflush(pop_charout_device); endif;
			    endif;
			else
			    ;;; this client is not subject to the watchdog mechanism at present, so enable interrupts again
			    ;;;true -> pop_asts_enabled;
			endif;
		    endprocedure); 
    endrepeat;    
enddefine;

;;; this will schedule the experiments according to priority and availability of hosts
define :method scheduleclients(es:experimentserver);
    lvars ex, availhosts;
    ;;; this should run forever in interactive mode, in batch mode the scheduler can quit once all experiments are scheduled
    while interactiveMODE do
	if verbose then npr('Scheduler: checking for queued experiments'); endif;
    	;;; check if there are more experiments that need to be run
    	while (nextrunnableexperiment(es) ->> ex) do
	    ;;; get the clients of the experiment
 	    ;;; MS 06-18-03: this is a procedure, needs to be applied to produce clients first
	    (ex.clients)() -> ex.clients;
	    lvars cp = ex.clients;
	    ;;; MS 06-18-03 add clients here to global client list (will make watchdog run more efficiently)            
            applist(ex.clients,procedure(cl);"SCHEDULED" -> cl.status;cl -> (es.allclients)(cl.CID);endprocedure);
	    ;;; and schedule them all
	    until null(cp) do
		;;; get an available host
	     	lvars host = getavailablehost(es,false),
		     cobject = hd(cp),
		     cname = cobject.CID,;

		;;; if no client from the experiment has been started, mark the experiment as started
		unless ex.starttime then		    
	    	    sysdaytime() -> ex.starttime;
	    	    if verbose then npr('Starting clients from experiment ' >< ex.name); endif;
		endunless;
		;;; now start the client in a separate thread
	    	startclient(es,cname,cobject,host,10);
                ;;; to avoid "rush" when a new experiment with lots of clients is submitted, which can lead to failures in initialization
		sleep_thread(SCHEDULETIME);	    	
		;;; take the client of the list
	    	tl(cp) -> cp;
	    enduntil;
	    ;;; mark experiment as running
	    "RUNNING" -> ex.status;
    	endwhile;
    	if verbose then npr('Scheduler: no queued experiments left'); endif;
	;;; MS: 06-18-03 clear the client property and reset the two gensym properties as there are no clients around here
        ;;; TODO: should also delete these items from the dictionary...
        if null(es.runningexps) then
/*            1 -> gensym("cl");
            if not(exists_group_thread("serveclient")) then
                1 -> gensym("serveclient");
            endif;
*/
            clearproperty(es.allclients);
        endif;
    	if interactiveMODE then
	    ;;; sleep for 10 sec. then check again
	    sleep_thread(10);
    	endif;
    endwhile;
    if verbose then npr('Scheduler finished'); endif;
enddefine;

;;; adds one or more new experiments to the queue and returns any error string (or false if there was none)
define :method enqueueexperiment(es:experimentserver,item);
    ;;; create a new experiment, let it set up possible other experiments
    lvars l = initexperiment(newexperiment(),item);
    ;;; check if it is false, then there was a problem, otherwise schedule the experiment(s)
    if isstring(l) then
	return(l);
    else
    	applist(l,
	    	procedure(e);
    		    dlocal pop_asts_enabled = false;
                    ;;; MS: 06-18-03 taken out for efficiency reasons, done in scheduleclient
		    ;;; add all the clients from the experiment to the global client list and mark them as scheduled
            	    ;;;applist(e.clients,procedure(cl);"SCHEDULED" -> cl.status;cl -> (es.allclients)(cl.CID);endprocedure);
    	    	    ;;; add the experiment to the global experiment list according to priority
    	    	    (es.experiments)(e.priority) <> [% e %] -> (es.experiments)(e.priority);
    	    	    "SCHEDULED" -> e.status;
	    	endprocedure);
	return(false);
    endif;
enddefine;

;;; this initializes the experiment server, it loads the host information from all host files
;;; and starts checking them, then it loads the experiment descriptions (the format is just
;;; an extension of the simworld parameterlist format)
define :method initexperimentserver(es:experimentserver,hostfiles,expfiles);
    lvars item;

    ;;; FOR WEB DEBUGGING, SHOULD BE REMOVED AT SOME POINT...
    ;;; Adding this here so that we can have access to the experiment server from
    ;;; the web interface; we need this in order to restart the watchdog process.
    ;;; if exists_thread('clientwatchdog') then 'true' else 'false' endif;
    ;;; start_thread('clientwatchdog', 10, clientwatchdog, [% ES %]); 'done';
    es -> ES;

    ;;; initialize paths and generate the testhostscript
    setuppaths();
    if generateTESTHOSTSCRIPT then
    	generatetesthostscript();
    endif;

    ;;; compile and start the web server if in interactive mode
    if interactiveMODE then
    	if verbose then npr('Initializing experiment server in interactive mode'); endif;
	compile(this_dir dir_>< 'webserver.p');
  	if verbose then npr('Starting web server at port '>< comm_port >< '...'); endif;
	start_thread('webserver',
		 100,
		 procedure();
		     ;;; keep accepting 1M connections
	             ;;; needs to be killed explicitly
    		     lvars comm_socket = sys_socket(`i`, `S`, false);
    		     [* ^comm_port] -> sys_socket_name(comm_socket, 1000000);		
		     if verbose then npr('Web server running at port '>< comm_port >< '...'); endif;
		     repeat
    			 webrequest(sys_socket_accept(es,comm_socket, false),serverweb);
    		     endrepeat;
		 endprocedure,
		 []);
    else
	if verbose then npr('Initializing experiment server in batch mode'); endif;
    endif;

    ;;; load all the host files
    if islist(hostfiles) then
	for item in hostfiles do
	    loadhosts(es,item);
	endfor;
    else
   	loadhosts(es,hostfiles);
    endif;
    if verbose then npr('Host information loaded.  Start updating host information...'); endif;

    ;;; now create a control socket, `S` = SOCK_STREAM) ASCII `i` = IP
    ;;; to listen to the client processes...
    sys_socket(`i`, `S`, false) -> es.control_sock;
    [* ^portnumber] -> sys_socket_name(es.control_sock,10000000);
    start_thread('clientserver',
		 10,
		 procedure();
		     if verbose then npr('Starting client server...'); endif;
		     ;;; keep accepting connections till all experiments have finished
	             ;;; needs to be killed explicitly
		     while true do
			 ;;; start a server thread to serve that client
			 if verbose then npr('Starting new serveclient thread'); endif;
		     	 start_thread(gensym("serveclient")><'',10,serveclient,
                          	      [% es,sys_socket_accept(es.control_sock,false) %]);
		     endwhile;
		     npr('Ack! clientserver should never get here!');
		 endprocedure,
		 []);
    if verbose then npr('Created socket for clients, waiting for client connect...'); endif;

    unless null(expfiles) then
    	;;; now create all the experiments and their initial clients
    	for item in expfiles do
	    ;;; each item is a file, so compile it,
	    ;;; create a new experiment and put it in the queue
	    lvars experror;
	    if (enqueueexperiment(es,compile(item))->>experror) then
		npr('Problem with experiment: ' >< experror);
	    endif;
    	endfor;
    	if verbose then npr('Experiment information loaded'); endif;
    	if verbose then npr('Waiting ten seconds for hosts to come online'); endif;
	;;; MS: use thread sleep here
    	sleep_thread(10);
    elseif interactiveMODE then
	if verbose then npr('No experiments submitted.  New experiments can be submitted through the web server'); endif;
    else
	mishap('No experiments submitted in batch mode ',[]);
    endunless;

    ;;; start experiments in order of priority (highest first)
    ;;; run this as a thread of highest priority
    start_thread('scheduler',
		 100,
		 scheduleclients,
		 [% es %]);
    if verbose then npr('Scheduler is running...'); endif;

    ;;; start client watch dog, this is to make sure that clients don't die, this will restart them if necessary
    ;;; run this as a thread of highest priority
    start_thread('clientwatchdog',
		 10,
		 clientwatchdog,
		 [% es %]);
    if verbose then npr('Client watchdog is running...'); endif;    
enddefine;

;;; this will start everything...
;;; expects a list of file names of hosts, and a list of experiments
;;; works with either version of the thread package (i.e., with or without fork turned on)
define setup(hostfiles,expfiles);
    ;;; create an experiment server and start it
	npr('\nSWAGES ' >< Version >< '\n(c) Matthias Scheutz 2003\nAll rights reserved.');
        start_thread('experimentserver',
		 100,
		 initexperimentserver,
		 [%newexperimentserver(),hostfiles,expfiles%]);
    ;;;if verbose then npr('Experiment scheduler is running...'); endif;
enddefine;


;;; start up the server program, if there are arguments, then the first is the hostsfile, the second
;;; can be any file that needs to be compiled before checking for whether the server should be started
;;; in interactive mode (if interactive mode is off and the there is not call to setup in the file passed
;;; as second argument, the server will not start)
if not(null(poparglist)) then
    ;;; check if there is also a hosts file, otherwise use the default hosts file
    poparglist(1) -> hostsfile;
    unless null(tl(poparglist)) then
 	compile(poparglist(2));
    endunless;
endif;

;;; now compile the server configuration file if it exists
if sys_file_exists(serverconfig) then
    compile(serverconfig);
endif;

;;; if no arguments are passed and interactive mode is on, then start the server in web mode
;;; with the default hosts file and no xperiments
if interactiveMODE then
    setup(hostsfile,[]);
endif;
