VmApps: wrapperWeir.cpp

// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2008 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License
// along with BOINC.  If not, see <http://www.gnu.org/licenses/>.

// vmwrapper.C
// VMWare wrapper program - lets you use BOINC to drive a VMWare Server
// guest OS

#include <stdio.h>
#include <vector>
#include <string>
#ifdef _WIN32
#include "boinc_win.h"
#include "win_util.h"
#else
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "procinfo.h"
#endif

#include "boinc_api.h"
#include "diagnostics.h"
#include "filesys.h"
#include "parse.h"
#include "str_util.h"
#include "util.h"
#include "error_numbers.h"

#include "vmware-vix/vix.h"

#define JOB_FILENAME "job.xml"
#define CHECKPOINT_FILENAME "checkpoint.txt"

#define POLL_PERIOD 1.0

using std::vector;
using std::string;

struct TASK
{
        string application;
        string stdin_filename;
        string stdout_filename;
        string stderr_filename;

        string vm;
        string innerjob;
        string datadir;
        string partial_credit;
        string snapshots;
        string user;
        string password;

        vector < string > data;
        vector < string > output;

        string checkpoint_filename;
        // name of task's checkpoint file, if any
        double checkpoint_cpu_time;
        // CPU time at last checkpoint
        string command_line;
        double weight;
        // contribution of this task to overall fraction done
        double final_cpu_time;
        double starting_cpu;
        // how much CPU time was used by tasks before this in the job file
        bool    suspended;
        double wall_cpu_time;
        // for estimating CPU time on Win98/ME and Mac


        #ifdef _WIN32

        HANDLE pid_handle;
        DWORD pid;
        HANDLE thread_handle;
        struct _stat last_stat;                          // mod time of checkpoint file

        #else

        int pid;
        struct stat last_stat;

        #endif


        bool stat_first;
        int parse (XML_PARSER &);
        bool poll (int &status);
        int run (int argc, char **argv);
        void kill ();
        void stop ();
        void resume ();
        double cpu_time ();

        inline bool has_checkpointed ()
        {
                bool changed = false;

                if (checkpoint_filename.size () == 0)
                        return false;

                struct stat new_stat;

                int retval = stat (checkpoint_filename.c_str (), &new_stat);

                if (retval)
                        return false;

                if (!stat_first && new_stat.st_mtime != last_stat.st_mtime)
                {
                        changed = true;
                }

                stat_first = false;

                last_stat.st_mtime = new_stat.st_mtime;
                return changed;
        }
};

vector < TASK > tasks;
APP_INIT_DATA aid;
bool graphics = false;

int TASK::parse (XML_PARSER & xp)
{

        string this_data, this_output;
        char tag[1024], buf[8192], buf2[8192];

        bool is_tag;

        weight = 1;
        final_cpu_time = 0;
        stat_first = true;
        while (!xp.get (tag, sizeof (tag), is_tag))
        {
                if (!is_tag)
                {
                        fprintf (stderr, "SCHED_CONFIG::parse(): unexpected text %s\n",
                                tag);
                        continue;
                }
                if (!strcmp (tag, "/task"))
                {
                        return 0;
                }
                else if (xp.parse_string (tag, "application", application))
                        continue;

                else if (xp.parse_string (tag, "innerjob", innerjob))
                        continue;
                else if (xp.parse_string (tag, "vm", vm))
                        continue;
                else if (xp.parse_string (tag, "datadir", datadir))
                        continue;
                else if (xp.parse_string (tag, "partial_credit", partial_credit))
                        continue;
                else if (xp.parse_string (tag, "snapshots", snapshots))
                        continue;


                else if (xp.parse_string (tag, "data", this_data)) {
                        data.push_back(this_data);
                        continue;
                }

                else if (xp.parse_string (tag, "output", this_output)) {
                        output.push_back(this_output);
                        continue;
                }

                else if (xp.parse_string (tag, "user", user))
                        continue;
                else if (xp.parse_string (tag, "password", password))
                        continue;

                else if (xp.parse_string (tag, "stdin_filename", stdin_filename))
                        continue;
                else if (xp.parse_string (tag, "stdout_filename", stdout_filename))
                        continue;
                else if (xp.parse_string (tag, "stderr_filename", stderr_filename))
                        continue;
                else if (xp.parse_str (tag, "command_line", buf, sizeof (buf)))
                {
                        while (1)
                        {
                                char *p = strstr (buf, "$PROJECT_DIR");

                                if (!p)
                                        break;
                                strcpy (buf2, p + strlen ("$PROJECT_DIR"));
                                strcpy (p, aid.project_dir);
                                strcat (p, buf2);
                        }
                        command_line = buf;
                        continue;
                }
                else if (xp.
                        parse_string (tag, "checkpoint_filename",
                        checkpoint_filename))
                        continue;
                else if (xp.parse_double (tag, "weight", weight))
                        continue;
        }
        return ERR_XML_PARSE;
}


int parse_job_file ()
{
        MIOFILE mf;
        char tag[1024], buf[256];

        bool is_tag;

        boinc_resolve_filename (JOB_FILENAME, buf, 1024);
        FILE *f = boinc_fopen (buf, "r");

        if (!f)
        {
                fprintf (stderr, "can't open job file %s\n", buf);
                return ERR_FOPEN;
        }
        mf.init_file (f);
        XML_PARSER xp (&mf);

        if (!xp.parse_start ("job_desc"))
                return ERR_XML_PARSE;
        while (!xp.get (tag, sizeof (tag), is_tag))
        {
                if (!is_tag)
                {
                        fprintf (stderr, "SCHED_CONFIG::parse(): unexpected text %s\n",
                                tag);
                        continue;
                }
                if (!strcmp (tag, "/job_desc"))
                {
                        fclose (f);
                        return 0;
                }
                if (!strcmp (tag, "task"))
                {
                        TASK task;
                        int retval = task.parse (xp);

                        if (!retval)
                        {
                                tasks.push_back (task);
                        }
                }
        }
        fclose (f);
        return ERR_XML_PARSE;
}


#ifdef _WIN32
// CreateProcess() takes HANDLEs for the stdin/stdout.
// We need to use CreateFile() to get them.  Ugh.
//
HANDLE
win_fopen (const char *path, const char *mode)
{
        SECURITY_ATTRIBUTES sa;
        memset (&sa, 0, sizeof (sa));
        sa.nLength = sizeof (sa);
        sa.bInheritHandle = TRUE;

        if (!strcmp (mode, "r"))
        {
                return CreateFile (path,
                        GENERIC_READ,
                        FILE_SHARE_READ, &sa, OPEN_EXISTING, 0, 0);
        }
        else if (!strcmp (mode, "w"))
        {
                return CreateFile (path,
                        GENERIC_WRITE,
                        FILE_SHARE_WRITE, &sa, OPEN_ALWAYS, 0, 0);
        }
        else if (!strcmp (mode, "a"))
        {
                HANDLE
                        hAppend = CreateFile (path,
                        GENERIC_WRITE,
                        FILE_SHARE_WRITE, &sa, OPEN_ALWAYS, 0, 0);
                SetFilePointer (hAppend, 0, NULL, FILE_END);
                return hAppend;
        }
        else
        {
                return 0;
        }
}
#endif

void slash_to_backslash (char *p)
{
        while (1)
        {
                char *q = strchr (p, '/');

                if (!q)
                        break;
                *q = '\\';
        }
}


int TASK::run (int argct, char **argvt)
{
        string stdout_path, stdin_path, stderr_path;
        char app_path[1024], buf[256];

        strcpy (buf, application.c_str ());
        char *p = strstr (buf, "$PROJECT_DIR");

        if (p)
        {
                p += strlen ("$PROJECT_DIR");
                sprintf (app_path, "%s%s", aid.project_dir, p);
        }
        else
        {
                boinc_resolve_filename (buf, app_path, sizeof (app_path));
        }

        // Append wrapper's command-line arguments to those in the job file.
        //
        for (int i = 1; i < argct; i++)
        {
                command_line += argvt[i];
                if ((i + 1) < argct)
                {
                        command_line += string (" ");
                }
        }

        fprintf (stderr, "wrapper: running %s (%s)\n",
                app_path, command_line.c_str ());

        #ifdef _WIN32
        PROCESS_INFORMATION process_info;
        STARTUPINFO startup_info;
        string command;

        slash_to_backslash (app_path);
        memset (&process_info, 0, sizeof (process_info));
        memset (&startup_info, 0, sizeof (startup_info));
        command = string ("\"") + app_path + string ("\" ") + command_line;

        // pass std handles to app
        //
        startup_info.dwFlags = STARTF_USESTDHANDLES;
        if (stdout_filename != "")
        {
                boinc_resolve_filename_s (stdout_filename.c_str (), stdout_path);
                startup_info.hStdOutput = win_fopen (stdout_path.c_str (), "a");
        }
        if (stdin_filename != "")
        {
                boinc_resolve_filename_s (stdin_filename.c_str (), stdin_path);
                startup_info.hStdInput = win_fopen (stdin_path.c_str (), "r");
        }
        if (stderr_filename != "")
        {
                boinc_resolve_filename_s (stderr_filename.c_str (), stderr_path);
                startup_info.hStdError = win_fopen (stderr_path.c_str (), "a");
        }
        else
        {
                startup_info.hStdError = win_fopen (STDERR_FILE, "a");
        }

                                                                 // bInheritHandles
        if (!CreateProcess (app_path, (LPSTR) command.c_str (), NULL, NULL, TRUE,
                CREATE_NO_WINDOW | IDLE_PRIORITY_CLASS,
                NULL, NULL, &startup_info, &process_info))
        {
                return ERR_EXEC;
        }
        pid_handle = process_info.hProcess;
        pid = process_info.dwProcessId;
        thread_handle = process_info.hThread;
        SetThreadPriority (thread_handle, THREAD_PRIORITY_IDLE);
        #else
        int retval, argc;
        char progname[256];
        char *argv[256];
        char arglist[4096];
        FILE *stdout_file;
        FILE *stdin_file;
        FILE *stderr_file;

        pid = fork ();
        if (pid == -1)
        {
                boinc_finish (ERR_FORK);
        }
        if (pid == 0)
        {
                // we're in the child process here
                //
                // open stdout, stdin if file names are given
                // NOTE: if the application is restartable,
                // we should deal with atomicity somehow
                //
                if (stdout_filename != "")
                {
                        boinc_resolve_filename_s (stdout_filename.c_str (), stdout_path);
                        stdout_file = freopen (stdout_path.c_str (), "a", stdout);
                        if (!stdout_file)
                                return ERR_FOPEN;
                }
                if (stdin_filename != "")
                {
                        boinc_resolve_filename_s (stdin_filename.c_str (), stdin_path);
                        stdin_file = freopen (stdin_path.c_str (), "r", stdin);
                        if (!stdin_file)
                                return ERR_FOPEN;
                }
                if (stderr_filename != "")
                {
                        boinc_resolve_filename_s (stderr_filename.c_str (), stderr_path);
                        stderr_file = freopen (stderr_path.c_str (), "a", stderr);
                        if (!stderr_file)
                                return ERR_FOPEN;
                }
                // construct argv
                // TODO: use malloc instead of stack var
                //
                argv[0] = app_path;
                strlcpy (arglist, command_line.c_str (), sizeof (arglist));
                argc = parse_command_line (arglist, argv + 1);
                setpriority (PRIO_PROCESS, 0, PROCESS_IDLE_PRIORITY);
                retval = execv (app_path, argv);
                exit (ERR_EXEC);
        }
        #endif
        wall_cpu_time = 0;
        suspended = false;
        return 0;
}


bool TASK::poll (int &status)
{
        if (!suspended)
                wall_cpu_time += POLL_PERIOD;
        #ifdef _WIN32
        unsigned long
                exit_code;

        if (GetExitCodeProcess (pid_handle, &exit_code))
        {
                if (exit_code != STILL_ACTIVE)
                {
                        status = exit_code;
                        final_cpu_time = cpu_time ();
                        return true;
                }
        }
        #else
        int
                wpid,
                stat;
        struct rusage
                ru;

        wpid = wait4 (pid, &status, WNOHANG, &ru);
        if (wpid)
        {
                final_cpu_time =
                        (float) ru.ru_utime.tv_sec + ((float) ru.ru_utime.tv_usec) / 1e+6;
                return true;
        }
        #endif
        return false;
}


void TASK::kill ()
{
        #ifdef _WIN32
        TerminateProcess (pid_handle, -1);
        #else
        ::kill (pid, SIGKILL);
        #endif
}


void TASK::stop ()
{
        suspended = true;
}


void TASK::resume ()
{
        suspended = false;
}




double TASK::cpu_time ()
{
        #ifdef _WIN32
        FILETIME creation_time, exit_time, kernel_time, user_time;
        ULARGE_INTEGER tKernel, tUser;
        LONGLONG totTime;

        int retval = GetProcessTimes (pid_handle, &creation_time, &exit_time,
                &kernel_time,
                &user_time);

        if (retval == 0)
        {
                return wall_cpu_time;
        }

        tKernel.LowPart = kernel_time.dwLowDateTime;
        tKernel.HighPart = kernel_time.dwHighDateTime;
        tUser.LowPart = user_time.dwLowDateTime;
        tUser.HighPart = user_time.dwHighDateTime;
        totTime = tKernel.QuadPart + tUser.QuadPart;

        return totTime / 1.e7;
        #elif defined(__APPLE__)
        // There's no easy way to get another process's CPU time in Mac OS X
        //
        return wall_cpu_time;
        #else
        return linux_cpu_time (pid);
        #endif
}




// Support for multiple tasks.
// We keep a checkpoint file that says how many tasks we've completed
// and how much CPU time has been used so far
//
void write_checkpoint (int ntasks, double cpu)
{
        FILE *f = fopen (CHECKPOINT_FILENAME, "w");

        if (!f)
                return;
        fprintf (f, "%d %f\n", ntasks, cpu);
        fclose (f);
}


void read_checkpoint (int &ntasks, double &cpu)
{
        int nt;
        double c;

        ntasks = 0;
        cpu = 0;
        FILE *f = fopen (CHECKPOINT_FILENAME, "r");

        if (!f)
                return;
        int n = fscanf (f, "%d %lf", &nt, &c);

        fclose (f);
        if (n != 2)
                return;
        ntasks = nt;
        cpu = c;
}



void check_vm_result(VixError err, VixHandle hostHandle, string errorString)
{
        if (VIX_OK != err)
        {
                fprintf (stderr, "\n\n Error: %s\n",errorString.c_str());
                fprintf (stderr, "Error message: \"%s\"\n", Vix_GetErrorText (err, NULL));
                VixHost_Disconnect (hostHandle);
                boinc_finish(100);
        }
}

int main (int argc, char **argv)
{

        BOINC_OPTIONS options;
        int retval, ntasks;
        unsigned int i;
        double cpu, total_weight = 0, w = 0;

        for (i = 1; i < (unsigned int) argc; i++)
        {
                if (!strcmp (argv[i], "--graphics"))
                {
                        graphics = true;
                }
        }

        memset (&options, 0, sizeof (options));
        options.main_program = true;
        options.check_heartbeat = true;
        options.handle_process_control = true;
        if (graphics)
        {
                options.backwards_compatible_graphics = true;
        }

        // boinc_init_options(&options);

        fprintf (stderr, "vmwrapper: starting\n");

        boinc_get_init_data (aid);

        retval = parse_job_file ();
        if (retval)
        {
                fprintf (stderr, "can't parse job file: %d\n", retval);
                boinc_finish (retval);
        }

        read_checkpoint (ntasks, cpu);
        if (ntasks > (int) tasks.size ())
        {
                fprintf (stderr, "Checkpoint file: ntasks %d too large\n", ntasks);
                boinc_finish (1);
        }

        for (i = 0; i < tasks.size (); i++)
        {
                total_weight += tasks[i].weight;
        }




        VixHandle hostHandle = VIX_INVALID_HANDLE;
        VixHandle jobHandle = VIX_INVALID_HANDLE;
        VixError err;
        string vmxFilename;

        jobHandle = VixHost_Connect (VIX_API_VERSION, VIX_SERVICEPROVIDER_VMWARE_SERVER, NULL, 0, NULL, NULL, 0, VIX_INVALID_HANDLE, NULL, NULL);

        err = VixJob_Wait (jobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &hostHandle, VIX_PROPERTY_NONE);

        if (VIX_OK != err)
        {
                fprintf (stderr, "Unable to connect to server!\n");
                fprintf (stderr, "Error message: \"%s\"\n", Vix_GetErrorText (err, NULL));
                VixHost_Disconnect (hostHandle);
                boinc_finish(1);
        }

        fprintf (stderr, "Connected to server.\n");


        Vix_ReleaseHandle (jobHandle);
        jobHandle = VIX_INVALID_HANDLE;


        for (i = 0; i < tasks.size (); i++)
        {

                fprintf(stderr,"Processing task %d.\n", i);

                TASK & task = tasks[i];

                // w += task.weight;

                // this line is to do with cpu time storage
                // if ((int) i < ntasks)
                //      continue;

                double frac_done = w / total_weight;


                fprintf(stderr, "Task %d.\n========\n\n", i+1);


                fprintf(stderr, "VM file root is %s\n", task.vm.c_str ());

                string vm_file_in_1, vm_file_in_2;
                char vm_file_1[1024], vm_file_2[1024];

                string pwd = get_current_dir_name ();


                // VMWare Server 1 requires full path
                vm_file_in_1 = pwd + "/" + task.vm + ".vmx";
                vm_file_in_2 = pwd + "/" + task.vm + ".vmdk";

//              fprintf (stderr, "Input filenames are %s, %s\n",
//                      vm_file_in_1.c_str (), vm_file_in_2.c_str ());

                boinc_resolve_filename (vm_file_in_1.c_str (), vm_file_1, 1024);
                boinc_resolve_filename (vm_file_in_2.c_str (), vm_file_2, 1024);

//              fprintf (stderr, "Resolved to %s, %s\n", vm_file_1, vm_file_2);

                if (access (vm_file_1, R_OK))
                {
                        fprintf (stderr, "Unable to access VM file %s!\n",vm_file_1);
                        boinc_finish (1);
                }

                if (access (vm_file_2, R_OK))
                {
                        fprintf (stderr, "Unable to access VM file %s!\n",vm_file_2);
                        boinc_finish (1);
                }


                // Register VM
                jobHandle = VixHost_RegisterVM (hostHandle, vm_file_1, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to register VM.");
                fprintf (stderr, "Registered virtual machine with the server.\n");

                VixHandle vmHandle = VIX_INVALID_HANDLE;

                // Open VM
                jobHandle = VixVM_Open (hostHandle, vm_file_1, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &vmHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to open VM.");
                fprintf (stderr, "Got handle.\n");

                // Power on VM
                jobHandle = VixVM_PowerOn (vmHandle, 0, VIX_INVALID_HANDLE, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to power on VM.");
                fprintf (stderr, "VM is now on (if it wasn't already).\n");

                // Wait for guest OS to start
                jobHandle = VixVM_WaitForToolsInGuest (vmHandle, 0, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Did not get confirmation of VMWare Tools.");

                // Log in
                jobHandle = VixVM_LoginInGuest (vmHandle, task.user.c_str(), task.password.c_str(), 0, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to log in.");
                fprintf (stderr, "Logged in. Preparing job...\n");


                string guestfile = task.datadir +  "/" + task.innerjob;
                string st_hostfile = pwd + "/" + task.innerjob;
                char hostfile[1024];

                boinc_resolve_filename (st_hostfile.c_str (), hostfile, 1024);

                jobHandle = VixVM_CopyFileFromHostToGuest (vmHandle, hostfile, guestfile.c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to copy executable.");

                Vix_ReleaseHandle (jobHandle);

                // Changing permissions, do we really need to do this?
                jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/chmod", ("+x " + guestfile).c_str (), 0, VIX_INVALID_HANDLE, NULL, NULL);  
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Unable to change permissions.");

                Vix_ReleaseHandle (jobHandle);


                fprintf (stderr, "Copying in input...\n");

                for(int files=0;files<task.data.size();files++)
                {
                        fprintf(stderr,"Processing: %s\n",task.data[files].c_str());

                        // should use boinc_resolve_filename here
                        jobHandle = VixVM_CopyFileFromHostToGuest (vmHandle, (pwd + "/" + task.data[files]).c_str(), (task.datadir + "/" + task.data[files]).c_str(),
                                0, VIX_INVALID_HANDLE,  NULL, NULL);
                        err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                        check_vm_result(err, hostHandle, "Error copying in file.");

                        Vix_ReleaseHandle (jobHandle);

                }






                // Run the target program.
                fprintf (stderr, "Running main program: ");
                jobHandle = VixVM_RunProgramInGuest (vmHandle, guestfile.c_str (), "", 0, VIX_INVALID_HANDLE, NULL, NULL);

                Bool complete=false;

                task.starting_cpu = cpu;
                task.checkpoint_cpu_time = cpu;


                BOINC_STATUS status;
                VixHandle innerJobHandle;
                VixHandle snapshotHandle;

                while(!complete)
                {
                        if(boinc_time_to_checkpoint())
                                fprintf(stderr,"Time to checkpoint run here...\n");



                        boinc_get_status (&status);

                        if (status.no_heartbeat || status.quit_request || status.abort_request)
                        {
                                VixHost_Disconnect (hostHandle);
                                boinc_finish(0);
                        }


                        // contents of this control structure needs fixing, since suspend doesn't work
                        if (status.suspended)
                        {
                                if (!task.suspended)
                                {
                                        // innerJobHandle = VixVM_Suspend(vmHandle,0,VIX_INVALID_HANDLE,NULL,NULL);
                                        // err = VixJob_Wait(innerJobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &snapshotHandle, VIX_PROPERTY_NONE);
                                        // check_vm_result(err, hostHandle, "Error pausing VM.");
                                        task.stop();
                                }
                        }
                        else
                        {
                                if (task.suspended)
                                {
                                        // innerJobHandle = VixVM_Unpause(vmHandle,0,VIX_INVALID_HANDLE,NULL,NULL);
                                        // err = VixJob_Wait(innerJobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &snapshotHandle, VIX_PROPERTY_NONE);
                                        // check_vm_result(err, hostHandle, "Error unpausing VM.");
                                        task.resume ();
                                }
                        }


                        // send_status_message (task, frac_done);
                        // fprintf(stderr,"Partial credit check here...\n");

                        // double current_cpu_time = task.starting_cpu + task.cpu_time ();

                        // if (task.has_checkpointed ())
                        // {
                        //      task.checkpoint_cpu_time = current_cpu_time;
                        // }

                        // not sure how to do this yet
                        // boinc_report_app_status (current_cpu_time, task.checkpoint_cpu_time, frac_done);


                        boinc_sleep(POLL_PERIOD);


                        if(!task.suspended) {
                                err = VixJob_CheckCompletion(jobHandle, &complete);
                                check_vm_result(err, hostHandle, "Error polling for job completion.");
                        }
                }


                cpu += task.final_cpu_time;
                // write_checkpoint here is necessary to keep track of CPU time
                // we either use the time measured, or the time from within
                write_checkpoint (i + 1, cpu);

                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Error running target program.");
                fprintf (stderr, "done.\n");




                Vix_ReleaseHandle (jobHandle);

        
                fprintf(stderr, "Copying out output...\n");

                for(int files=0;files<task.output.size();files++)
                {
                        fprintf(stderr,"Processing: %s\n",task.output[files].c_str());


                        // FIXME: need to tell BOINC about these files
                        jobHandle = VixVM_CopyFileFromGuestToHost (vmHandle, (task.datadir + "/" + task.output[0]).c_str(), 
                                (pwd + "/" + task.output[files]).c_str (), 0, VIX_INVALID_HANDLE, NULL, NULL);
                        err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                        check_vm_result(err, hostHandle, "Error copying out file.");

                        Vix_ReleaseHandle (jobHandle);

                        jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  (task.datadir + "/" + task.output[files]).c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);       
                        err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                        check_vm_result(err, hostHandle, "Error deleting file.");

                        Vix_ReleaseHandle (jobHandle);
                }


                // now delete the executable, the output and the data
                fprintf(stderr, "Sanitising virtual machine...\n");

                for(int files=0;files<task.data.size();files++)
                {
                        fprintf(stderr,"Removing: %s\n",task.data[files].c_str());

                        jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  (task.datadir + "/" + task.data[files]).c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL); 
                        err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                        check_vm_result(err, hostHandle, "Error deleting file.");

                        Vix_ReleaseHandle (jobHandle);
                }


                fprintf(stderr,"Removing executable.\n");
                jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  guestfile.c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);       
                err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
                check_vm_result(err, hostHandle, "Error deleting file.");

                Vix_ReleaseHandle (jobHandle);

        

        }

        boinc_finish (0);
}


#ifdef _WIN32

int WINAPI
WinMain (HINSTANCE hInst, HINSTANCE hPrevInst, LPSTR Args, int WinMode)
{
        LPSTR command_line;
        char *argv[100];
        int argc;

        command_line = GetCommandLine ();
        argc = parse_command_line (command_line, argv);
        return main (argc, argv);
}
#endif