#!/usr/bin/perl
#Sumbit job for execution
package oarsub;
require Exporter;

use strict;

#use warnings;
use POSIX qw(strftime);
use Fcntl;
use Data::Dumper;
use Sys::Hostname;
use Getopt::Long;
use File::Basename;
use File::Temp qw/ tempfile /;
use Cwd;
use OAR::Conf qw(init_conf dump_conf get_conf is_conf);
use OAR::Sub;
use Capture::Tiny ':all';

my $Old_umask = sprintf("%lo", umask());
umask(oct("022"));

select(STDOUT);
$| = 1;

#Try to load XML module
my $XML_enabled = 1;
unless (eval "use XML::Dumper qw(pl2xml);1") {
    $XML_enabled = 0;
}

#Try to load YAML module
my $YAML_enabled = 1;
unless (eval "use YAML;1") {
    $YAML_enabled = 0;
}

#Try to load JSON module
my $JSON_enabled = 1;
unless (eval "use JSON;1") {
    $JSON_enabled = 0;
}

# suitable Data::Dumper configuration for serialization
$Data::Dumper::Purity   = 1;
$Data::Dumper::Terse    = 1;
$Data::Dumper::Indent   = 0;
$Data::Dumper::Deepcopy = 1;

#For interactive
my $remote_host;
my $remote_port;
my $Deploy_hostname;
my $Cosystem_hostname;
my $Deploy_cosystem_job_exec_system;
my $Cpuset_field;
my $Cpuset_path;
my $Host = hostname;

my $Server;
my $Job_id;
my $Job_id_list_ref;
my $Interactive = 0;
my $Reservation = "0";
my $Reservation_start;
my $Reservation_end;
my $Default_resources;
my $Nodes_resources;
my $connect_job;
my $Scan_script;
my @resource;
my $Queue_name;
my $Job_sql_properties = "";
my $Cmd_executor;
my $idFile = undef;
my $sos;
my $Checkpoint = 0;
my $notify;
my $job_name;
my $job_env;
my $job_hold;
my $Directory;
my $Directory_default = getcwd();
my @Type;
my @Anterior_job;
my $Checkpoint_signal;
my $Checkpoint_signal_default = 12;    # SIGUSR2
my $Stdout_file;
my $Stderr_file;
my $Resubmit;
my $Project;
my $Project_default = "default";
my $XML_mode;
my $YAML_mode;
my $JSON_mode;
my $DUMPER_mode;
my $use_job_key;
my $job_key_priv           = "";
my $job_key_pub            = "";
my $import_job_key_file    = "";
my $import_job_key_inline  = "";
my $export_job_key_file    = "";
my $Initial_request_string = "oarsub @ARGV";
my $Verbose_level;
my $lusr = $ENV{OARDO_USER};
my $Machine_parseable_output;
my $micheline_print_func;
my $micheline_warn_func;
my $micheline_capture_func;

# a non array-job is an array-job with 1 subjob
my $array_job_nb;
my $array_param_file;
my $array_params_ref;

# ignore ^Z signal
$SIG{'TSTP'} = 'IGNORE';

#to catch ^C signal
$SIG{INT}  = \&signal_handler;
$SIG{HUP}  = \&signal_handler;
$SIG{PIPE} = \&signal_handler;

# to address ^C in interactive submission
sub signal_handler($) {
    my $signal = shift;
    if (defined($Job_id_list_ref)) {
        if ($signal eq "INT") {
            print(STDERR "\n\n# Caught Interrupt (^C), ");
        }
        OAR::Sub::delete_jobs($Job_id_list_ref, $remote_host, $remote_port);
        OAR::Sub::close_db_connection();
        exit(1);
    }
}

sub oarsub_print($) {
    my $str = shift;

    if (!$Machine_parseable_output) {
        print($str);
    }
}

sub oarsub_warn($) {
    my $str = shift;

    if (!$Machine_parseable_output) {
        warn($str);
    }
}

# Connect to a job and give the shell of the user on the remote host
sub connect_job($$$) {
    my $job_id       = shift;
    my $stop_oarexec = shift;
    my $openssh_cmd  = shift;

    my $xauth_path = $ENV{OARXAUTHLOCATION};
    OAR::Sub::open_ro_db_connection();
    my $job = OAR::Sub::get_job($job_id);

    if ((($lusr eq $job->{job_user}) or ($lusr eq "oar")) && ($job->{state} eq "Running")) {
        my $types = OAR::Sub::get_current_job_types($job_id);

        #noop
        if (defined($types->{noop})) {
            oarsub_warn("# Error: it is not possible to connect to a NOOP job.\n");
            OAR::Sub::close_db_connection();
            exit(17);
        }
        my $hosts_tmp               = OAR::Sub::get_job_current_hostnames($job_id);
        my @hosts                   = @$hosts_tmp;
        my $host_to_connect_via_ssh = $hosts[0];

        #deploy, cosystem and no host part
        if ((defined($types->{cosystem})) or ($#hosts < 0)) {
            $host_to_connect_via_ssh = $Cosystem_hostname;
        } elsif (defined($types->{deploy})) {
            $host_to_connect_via_ssh = $Deploy_hostname;
        }

        #cpuset part
        if (
            (   defined($Cpuset_field)         and
                defined($Cpuset_path)          and
                (!defined($types->{cosystem})) and
                (!defined($types->{deploy}))   and
                ($#hosts >= 0))
        ) {
            $ENV{OAR_CPUSET} = $Cpuset_path . '/' . OAR::Sub::get_job_cpuset_name($job_id);
        } else {
            $ENV{OAR_CPUSET} = "";
        }
        my $moldable = OAR::Sub::get_current_moldable_job($job->{assigned_moldable_job});
        my $job_user = $job->{job_user};
        OAR::Sub::close_db_connection();
        my @passinfo = getpwnam($lusr) or
          die("Cannot retreive system information for user $lusr\n");
        my $shell = $passinfo[8];
        my $uid   = $passinfo[2];
        unless ((defined($xauth_path)) and
            (-x $xauth_path) and
            ($ENV{DISPLAY} =~ /^[\w.-]*:\d+(?:\.\d+)?$/)) {
            $ENV{DISPLAY} = "";
        }
        if ($ENV{DISPLAY} ne "") {
            if ($Verbose_level >= 1) {
                oarsub_print("# Initialize X11 forwarding...\n");
            }

            # first, get rid of remaining unused .Xautority.{pid} files...
            system(
                {"bash"} "bash",
                "-c",
                'for f in $HOME/.Xauthority.*; do [ -e "/proc/${f#$HOME/.Xauthority.}" ] || rm -f $f; done'
            );
            my $new_xauthority = $ENV{HOME} . "/.Xauthority.$$";
            system(
                {"bash"} "bash",
                "-c",
                '[ -x "' . $xauth_path .
                  '" ] && OARDO_BECOME_USER=' . $lusr . ' oardodo bash --noprofile --norc -c "' .
                  $xauth_path . ' extract - ${DISPLAY/#localhost:/:}" | XAUTHORITY=' .
                  $new_xauthority . ' ' . $xauth_path . ' -q merge - 2>/dev/null');
            $ENV{XAUTHORITY} = $new_xauthority;
        }
        my $node_file   = OAR::Sub::get_default_oarexec_directory() . "/$job_id";
        my $res_file    = OAR::Sub::get_default_oarexec_directory() . "/$job_id" . "_resources";
        my $oarsub_pids = OAR::Sub::get_oarsub_connections_file_name($job_id);
        my %params      = (
            "node_file"           => $node_file,
            "job_id"              => $job_id,
            "array_id"            => $job->{array_id},
            "array_index"         => $job->{array_index},
            "user"                => $lusr,
            "shell"               => $shell,
            "launching_directory" => $job->{launching_directory},
            "resource_file"       => $res_file,
            "job_name"            => $job->{job_name},
            "job_project"         => $job->{project},
            "job_walltime"        => OAR::Sub::duration_to_sql($moldable->{moldable_walltime}),
            "job_walltime_sec"    => $moldable->{moldable_walltime},
            "job_env"             => $job->{job_env});
        my $str = OAR::Sub::get_oarexecuser_script_for_oarsub(\%params);
        my ($cmd_name, @cmd_opts) = split(" ", $openssh_cmd);
        my @cmd;
        my $i = 0;
        $cmd[$i] = $cmd_name;
        $i++;

        foreach my $p (@cmd_opts) {
            $cmd[$i] = $p;
            $i++;
        }
        if ($ENV{OAR_CPUSET} ne "") {
            $cmd[$i] = "-oSendEnv=OAR_CPUSET";
            $i++;
        }
        if ($ENV{DISPLAY} ne "") {
            $cmd[$i] = "-X";
        } else {
            $cmd[$i] = "-x";
        }
        $i++;
        $cmd[$i] = "-t";
        $i++;
        $cmd[$i] = $host_to_connect_via_ssh;
        $i++;
        $str =~ s/\n//g;
        my $interactive_job_hook_file = get_conf("INTERACTIVE_JOB_HOOK_EXEC_FILE");
        my $interactive_job_hook_cmd;

        if (defined($interactive_job_hook_file)) {
            $interactive_job_hook_cmd =
              "if test -x $interactive_job_hook_file; then oardodo $interactive_job_hook_file \$\$,\$PPID:$lusr:$job_id:$job->{name}:$job->{project}:$moldable->{moldable_walltime}; else true; fi && ";
        }

        my $oardodo_cmd = "OARDO_BECOME_USER=$job_user oardodo bash --noprofile --norc -c '$str'";
        if (defined($types->{cosystem}) or defined($types->{deploy}) or ($#hosts < 0)) {
            if ($Deploy_cosystem_job_exec_system eq "systemd-run") {
                $oardodo_cmd =
                  "oardodo systemd-run -q --uid=$uid --pty --wait --collect --service-type=exec --slice=user-$uid.slice /bin/bash --noprofile --norc -c '$str'";
            } elsif ($Deploy_cosystem_job_exec_system eq "none") {

                # Use regular command by default
            }
        }

        if ($ENV{DISPLAY} ne "") {
            $cmd[$i] =
              "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && ($xauth_path -q extract - \${DISPLAY/#localhost:/:} | OARDO_BECOME_USER=$lusr oardodo $xauth_path merge -) && [ \"$lusr\" != \"$job_user\" ] && OARDO_BECOME_USER=$lusr oardodo bash --noprofile --norc -c \"chmod 660 \\\$HOME/.Xauthority\" ;TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && $oardodo_cmd";
            $i++;
        } else {
            $cmd[$i] =
              "bash -c '${interactive_job_hook_cmd}echo \$PPID >> $oarsub_pids && TTY=\$(tty) && test -e \$TTY && oardodo chown $job_user:oar \$TTY && oardodo chmod 660 \$TTY' && $oardodo_cmd";
            $i++;
        }

        #essential: you become oar instead of the user
        #UID=EUID
        $< = $>;
        if ($Verbose_level >= 1) {
            oarsub_print("# Connect to OAR job $job_id via node $host_to_connect_via_ssh\n");
        }
        system({ $cmd[0] } @cmd);
        my $exit_value = $? >> 8;
        if ($exit_value == 2) {
            oarsub_warn(
                "# Error: cannot go into working directory: $job->{launching_directory}.\n");
        } elsif ($exit_value == 255) {
            oarsub_warn("# Error: job was terminated.\n");
        } elsif ($exit_value != 0) {
            oarsub_warn("# Error: an unknown error occured ($?).\n");
        }
        if ($stop_oarexec > 0) {
            my %params = (
                "host"         => $host_to_connect_via_ssh,
                "job_id"       => $job_id,
                "signal"       => "USR1",
                "time_to_wait" => 0,
                "ssh_cmd"      => $openssh_cmd);
            OAR::Sub::signal_oarexec(\%params);
        }
        oarsub_print("Disconnected from OAR job $job_id.\n");
    } else {
        if ($job->{state} ne "Running") {
            oarsub_warn(
                "# Error: job $job_id is not running. Its current state is $job->{state}.\n");
        }
        if (($lusr ne $job->{job_user}) and ($lusr ne "oar")) {
            oarsub_warn(
                "# Error: you are not the right user for job $job_id. This job is owned by $job->{job_user}.\n"
            );
        }
        OAR::Sub::close_db_connection();
        return (20);
    }
    return (0);
}

#Print help message
sub usage() {
    print <<EOS;
Usage: $0 [options] [-I|-C [<job id>]|-r <start date>[, <end date>]|<script>]
Submit a job the OAR batch scheduler
Options are:
 -I, --interactive             Request an interactive job.
 -C, --connect[=<job id>]      Connect to a running job. Job id may be omitted
                               if only one job is running.
 -l, --resource=<description>[,<walltime>]
                               Set the description of the requested resources
                               for the job and a walltime which specifies an
                               estimation of the maximum duration of the job
                               (the job process can terminate earlier).
                               Ex: host=4/cpu=1,walltime=2:00:00
                               Option can be used several times, to define a
                               moldable job: a job accepting different
                               combination of resource definitions.
     --array <number>          Specify an array job with 'number' subjobs
     --array-param-file <file> Specify an array job on which each subjob will
                               receive one line of the file as parameter.
 -S, --scanscript              Request oarsub to scan the script file given as
                               the job command, for extra options to apply to
                               the job (lines starting with #OAR).
 -q, --queue=<queue>           Set the queue to submit the job to.
 -p, --property="<SQL WHERE CLAUSE>"
                               Add constraints to the wanted resource for the
                               job, using the resources properties.
                               Constraints use the SQL WHERE CLAUSE syntax.
 -r, --reservation="<start date|now>[, <end date>]"
                               Request that the job starts (and optionally
                               ends) at a specified time. Using this option,
                               the job is called an advance reservation.
                               Start and end dates use the YYYY-MM-DD hh:mm:ss
                               format. If YYYY-MM-DD is omitted, it defaults to
                               the current day. In hh:mm:ss, ss and mm can be
                               omitted. Quotes are only required when spaces
                               are present. When an end date is provided the
                               job walltime is inferred, unless provided in the
                               -l resources request which prevails. The special
                               keyword now can be used as the start date to
                               request an advance reservation job that starts
                               just now.
     --checkpoint=<delay>      Enable the checkpointing for the job. A signal
                               is sent DELAY seconds before the walltime on
                               the first processus of the job.
     --signal=<#sig>           Specify the signal to use when checkpointing
                               Use signal numbers, default is 12 (SIGUSR2).
 -t, --type=<type>             Specify a specific type (deploy, besteffort,
                               cosystem, checkpoint, timesharing). Option can
                               be specified several times.
 -d, --directory=<dir>         Specify the directory where to launch the
                               command (default is current directory).
     --project=<txt>           Specify a name of a project the job belongs to.
 -n, --name=<txt>              Specify an arbitrary name for the job.
 -a, --anterior=<job id>       Anterior job that must be terminated to start
                               this new one.
     --notify=<txt>            Specify a notification method
                               (mail or command to execute). Ex:
                                   --notify "mail:name\@domain.com"
                                   --notify "exec:/path/to/script args"
     --resubmit=<job id>       Resubmit the given job as a new one.
 -k, --use-job-key             Activate the job-key mechanism.
 -i, --import-job-key-from-file=<file>
                               Import the job-key to use from a files instead
                               of generating a new one.
     --import-job-key-inline=<txt>
                               Import the job-key to use inline instead of
                               generating a new one.
 -e  --export-job-key-to-file=<file>
                               Export the job key to a file. Warning: the
                               file will be overwritten if it already exists.
                               (the %jobid% pattern is automatically replaced)
 -O  --stdout=<file>           Specify the file that will store the standart
                               output stream of the job.
                               (the %jobid% pattern is automatically replaced)
 -E  --stderr=<file>           Specify the file that will store the standart
                               error stream of the job.
                               (the %jobid% pattern is automatically replaced)
     --hold                    Set the job state into Hold instead of Waiting,
                               so that it is not scheduled (you must run
                               "oarresume" to turn it into the Waiting state).
 -D, --dumper                  Print result in DUMPER format.
 -X, --xml                     Print result in XML format.
 -Y, --yaml                    Print result in YAML format.
 -J, --json                    Print result in JSON format.
 -h, --help                    Print this help message.
 -V, --version                 Print OAR version number.
 -v,                           Verbose mode, oarsub and admissions rules (if
                               compatible) will print more information on
                               terminal. Multiple -v can be used to increase
                               the level of verbosity (max: 2).
EOS
}

# Parse -l options and return an array of hashtables with resources for a moldable job
sub parse_resource_descriptions($) {
    my $resource_ref = shift;

    my @resource = @{$resource_ref};
    if ($#resource < 0) {
        push(@resource, $Default_resources);
    }

    #print "--@resource--\n";

    my @result;
    foreach my $r (@resource) {
        my @resource_groups;
        my $end_loop = 0;
        while ($end_loop == 0) {
            my $initial_resource = $r;
            my %tmp_result;
            if ($r =~ /^\s*(\++|\,+|\s*)\s*\{(.+?)}(.*)$/) {

                # $1 = property string
                $tmp_result{property} = $2;
                $r = $3;
            }

            my $resources_to_parse;
            if (($r =~ /^\s*(\++|\,+|\s*)\s*[\/]*([^\,\+]+)\s*(.*)$/) and ($2 !~ /^\s*walltime/)) {
                $resources_to_parse = $2;
                $r                  = $3;
            } else {
                $Default_resources =~ /^\s*[\/]*(.+)$/;

                # Remove first /
                $resources_to_parse = $1;
            }
            my @slash_split = split('\/', $resources_to_parse);
            my @resources_list;
            foreach my $l (@slash_split) {
                if ($l =~ /^\s*(\w+)\s*=\s*(\d+)\s*$/) {
                    my $tmp = $1;
                    $tmp = $Nodes_resources if ($tmp eq "nodes");
                    push(@resources_list, { resource => $tmp, value => $2 });
                } elsif ($l =~ /^\s*(\w+)\s*=\s*ALL\s*$/) {
                    my $tmp = $1;
                    $tmp = $Nodes_resources if ($tmp eq "nodes");
                    push(@resources_list, { resource => $tmp, value => -1 });
                } elsif ($l =~ /^\s*(\w+)\s*=\s*BESTHALF\s*$/) {
                    my $tmp = $1;
                    $tmp = $Nodes_resources if ($tmp eq "nodes");
                    push(@resources_list, { resource => $tmp, value => -3 });
                } elsif ($l =~ /^\s*(\w+)\s*=\s*BEST\s*$/) {
                    my $tmp = $1;
                    $tmp = $Nodes_resources if ($tmp eq "nodes");
                    push(@resources_list, { resource => $tmp, value => -2 });
                } else {
                    oarsub_warn("# Error: cannot recognize the resource description: $l\n");
                    exit(18);
                }
            }
            $tmp_result{resources} = \@resources_list;

            if ($r =~ /^[\s\,]*walltime\s*=\s*([\d|:]+)\s*$/) {

                #walltime part
                my ($w_h, $w_mn, $w_sec) = split(':', $1);
                if (defined($w_h)) {
                    if (not defined($w_mn)) {
                        $resource_groups[1] = "$w_h:00:00";
                    } elsif (not defined($w_sec)) {
                        $resource_groups[1] = "$w_h:$w_mn:00";
                    } else {
                        $resource_groups[1] = "$w_h:$w_mn:$w_sec";
                    }
                    $resource_groups[1] = OAR::Sub::sql_to_duration("$resource_groups[1]");
                } else {
                    oarsub_warn("# Error: cannot recognize walltime resource value\n");
                    exit(18);
                }
                $r = $2;
            }
            if ($r eq $initial_resource) {
                oarsub_warn("# Error: cannot recognize -- $r -- resource\n");
                exit(18);
            }
            push(@{ $resource_groups[0] }, \%tmp_result);

            if ($r =~ /^\s*$/) {
                $end_loop = 1;
            }
        }
        push(@result, \@resource_groups);
    }

    return (@result);
}

sub parse_reservation($$) {
    my $interactive   = shift;
    my $resource_list = shift;
    my $server_port;
    my $time_format = "(0?[0-9]|1[0-9]|2[0-3])(?::(0?[0-9]|[1-5][0-9])(?::(0?[0-9]|[1-5][0-9]))?)?";
    my $time_format_regex = qr/^\s*$time_format\s*$/;
    my $date_format_regex =
      qr/^\s*(\d{4}\-(?:0?[0-9]|1[0-2])\-(?:0?[1-9]|[1-2][0-9]|3[0-1]))\s+$time_format\s*$/;

    if ($Reservation ne "0") {

        my @guess;

        #Split the Reservation into the start and end time, then test if the syntax is right
        ($Reservation_start, $Reservation_end) = split(/\s*,\s*/, $Reservation);
        chomp($Reservation_start);
        if ($Reservation_start =~ m/$date_format_regex/m) {
            $Reservation_start = OAR::Sub::sql_to_local("$1 $2 $3 $4");
        } else {
            my $res_str;
            if ($Reservation_start =~ m/$time_format_regex/m) {
                my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
                $res_str = OAR::Sub::ymdhms_to_sql($year, $mon, $mday, $1, $2, $3);
            } elsif ($Reservation_start eq "now") {
                my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime(time);
                $res_str = OAR::Sub::ymdhms_to_sql($year, $mon, $mday, $hour, $min, $sec);
            } else {
                oarsub_warn(
                    "# Error: syntax error in <start date> for the -r or --reservation option. It takes a date as argument with the format: \"YYYY-MM-DD hh:mm:ss\".\n"
                );
                OAR::Sub::close_db_connection;
                exit(7);
            }
            push(@guess, "start: $res_str");
            $Reservation_start = OAR::Sub::sql_to_local("$res_str");
        }

        if ($Reservation_end ne undef) {

            #Test if this job has a reservation end and the syntax is right
            if ($Reservation_end =~ m/$date_format_regex/m) {
                $Reservation_end = OAR::Sub::sql_to_local("$1 $2 $3 $4");
            } else {
                my $res_str;
                if ($Reservation_end =~ m/$time_format_regex/m) {
                    my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) =
                      localtime(time);
                    $res_str         = OAR::Sub::ymdhms_to_sql($year, $mon, $mday, $1, $2, $3);
                    $Reservation_end = OAR::Sub::sql_to_local("$res_str");
                    if ($Reservation_end <= $Reservation_start) {
                        $res_str = OAR::Sub::ymdhms_to_sql($year, $mon, $mday + 1, $1, $2, $3);
                        $Reservation_end = OAR::Sub::sql_to_local("$res_str");
                    }
                } else {
                    oarsub_warn(
                        "# Error: syntax error in <end date> for the -r or --reservation option. It takes a date as argument with the format: \"YYYY-MM-DD hh:mm:ss\".\n"
                    );
                    OAR::Sub::close_db_connection();
                    exit(7);
                }
                push(@guess, "end: $res_str");
            }

            my $reservation_walltime = $Reservation_end - $Reservation_start;
            push(@guess,
                "(computed walltime: " . OAR::Sub::duration_to_sql($reservation_walltime) . ")");

            foreach my $list (@{$resource_list}) {
                if ($list->[1] eq undef) {
                    $list->[1] = $reservation_walltime;
                } else {
                    oarsub_warn(
                        "# Warning: ignore reservation end date because walltime is set in resource description.\n"
                    );
                }
            }
        }
        if (@guess and $Verbose_level >= 1) {
            oarsub_print("# Guessed reservation " . join(" ", @guess) . ".\n");
        }
        if ($interactive eq 0) {
            ($Server, $server_port) = OAR::Sub::init_tcp_server();
        }
    }

    if ($interactive eq 1) {
        ($Server, $server_port) = OAR::Sub::init_tcp_server();
    }

    return ($Server, $server_port);
}

sub run_advance_reservation_validation_hook($$) {
    my $hook   = shift;
    my $status = shift;
    my $ret    = system("$hook $Job_id_list_ref->[0] $status");

    if ($ret != 0) {
        oarsub_warn("# Error: an error occured in the advance reservation validation hook\n");
        exit(1);
    }
}

#
# Main
#

init_conf($ENV{OARCONFFILE});
$remote_host = get_conf("SERVER_HOSTNAME");
$remote_port = get_conf("SERVER_PORT");

$Default_resources = get_conf("OARSUB_DEFAULT_RESOURCES");
if (!defined($Default_resources)) {
    $Default_resources = "/resource_id=1";
}

$Nodes_resources = get_conf("OARSUB_NODES_RESOURCES");
if (!defined($Nodes_resources)) {
    $Nodes_resources = "resource_id";
}

$Deploy_hostname = get_conf("DEPLOY_HOSTNAME");
if (!defined($Deploy_hostname)) {
    $Deploy_hostname = $remote_host;
}

$Cosystem_hostname = get_conf("COSYSTEM_HOSTNAME");
if (!defined($Cosystem_hostname)) {
    $Cosystem_hostname = $remote_host;
}

$Deploy_cosystem_job_exec_system = get_conf("DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM");
if (!defined($Deploy_cosystem_job_exec_system)) {
    $Deploy_cosystem_job_exec_system = "none";
}
if (($Deploy_cosystem_job_exec_system ne "none") and
    ($Deploy_cosystem_job_exec_system ne "systemd-run")) {
    die("[ERROR] Bad configuration for DEPLOY_COSYSTEM_JOB_EXEC_SYSTEM: '$Deploy_cosystem_job_exec_system' is unsupported\n"
    );
}

$Cpuset_field = get_conf("JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD");
$Cpuset_path  = get_conf("CPUSET_PATH");

if (is_conf("OAR_RUNTIME_DIRECTORY")) {
    OAR::Sub::set_default_oarexec_directory(get_conf("OAR_RUNTIME_DIRECTORY"));
}
my $default_oar_dir = OAR::Sub::get_default_oarexec_directory();
if (!(((-d $default_oar_dir) and (-O $default_oar_dir)) or (mkdir($default_oar_dir)))) {
    die("# Error: cannot create the OAR directory $default_oar_dir or bad permissions.\n");
}

my $binpath;
if (defined($ENV{OARDIR})) {
    $binpath = $ENV{OARDIR} . "/";
} else {
    die("# Error: OARDIR env variable must be defined.\n");
}

my $Openssh_cmd = get_conf("OPENSSH_CMD");
$Openssh_cmd = OAR::Sub::get_default_openssh_cmd() if (!defined($Openssh_cmd));

if (is_conf("OAR_SSH_CONNECTION_TIMEOUT")) {
    OAR::Sub::set_ssh_timeout(get_conf("OAR_SSH_CONNECTION_TIMEOUT"));
}

Getopt::Long::Configure("gnu_getopt");
my $Version;
my @Verbose;

GetOptions(
    "resource|l=s"                 => \@resource,
    "queue|q=s"                    => \$Queue_name,
    "interactive|I"                => \$Interactive,
    "property|p=s"                 => \$Job_sql_properties,
    "reservation|r=s"              => \$Reservation,
    "connect|C:i"                  => \$connect_job,
    "checkpoint=i"                 => \$Checkpoint,
    "help|h"                       => \$sos,
    "notify=s"                     => \$notify,
    "type|t=s"                     => \@Type,
    "directory|d=s"                => \$Directory,
    "name|n=s"                     => \$job_name,
    "project=s"                    => \$Project,
    "hold"                         => \$job_hold,
    "array=i"                      => \$array_job_nb,
    "array-param-file=s"           => \$array_param_file,
    "anterior|a=i"                 => \@Anterior_job,
    "signal=i"                     => \$Checkpoint_signal,
    "stdout|O=s"                   => \$Stdout_file,
    "stderr|E=s"                   => \$Stderr_file,
    "resubmit=i"                   => \$Resubmit,
    "scanscript|S"                 => \$Scan_script,
    "xml|X"                        => \$XML_mode,
    "yaml|Y"                       => \$YAML_mode,
    "json|J"                       => \$JSON_mode,
    "dumper|D"                     => \$DUMPER_mode,
    "use-job-key|k"                => \$use_job_key,
    "import-job-key-inline-priv=s" => \$import_job_key_inline,
    "import-job-key-from-file|i=s" => \$import_job_key_file,
    "export-job-key-to-file|e=s"   => \$export_job_key_file,
    "version|V"                    => \$Version,
    "v"                            => \@Verbose
  ) or
  exit(1);

$Machine_parseable_output =
  ((defined($DUMPER_mode)) or (defined($YAML_mode) or ($XML_mode) or ($JSON_mode)));

if ($Machine_parseable_output) {
    $micheline_print_func   = sub { };
    $micheline_warn_func    = sub { };
    $micheline_capture_func = \&capture;
} else {
    $micheline_print_func   = sub { print(@_) };
    $micheline_warn_func    = sub { warn(@_) };
    $micheline_capture_func = \&tee;
}

$Verbose_level = scalar @Verbose;
if ($Verbose_level > 2) {
    $Verbose_level = 2;
}

if (defined($Version)) {
    oarsub_print("OAR version: " . OAR::Sub::get_oar_version() . "\n");
    exit(0);
}

if (defined($sos)) {
    usage();
    exit(0);
}

# Check the default name of the key if we have to generate it
if (is_conf("OARSUB_FORCE_JOB_KEY")) {
    if (lc(get_conf("OARSUB_FORCE_JOB_KEY")) eq "yes") {
        $use_job_key = 1;
    }
}

# If OAR_JOB_KEY_FILE is set in the shell environment, then imply use_job_key
# because oarsh will use OAR_JOB_KEY_FILE as well and fail if the job is
# setup without a job_key
if (defined($ENV{OAR_JOB_KEY_FILE})) {
    $use_job_key = 1;
}

OAR::Sub::open_db_connection();

if (defined($Resubmit)) {
    if ($Verbose_level >= 1) {
        oarsub_print("# Resubmitting job $Resubmit...");
    }
    my $err = OAR::Sub::resubmit_job($Resubmit);
    OAR::Sub::close_db_connection();
    if ($err > 0) {
        $Job_id = $err;
        if ($Verbose_level >= 1) {
            oarsub_print("# DONE\n");
        }
        if ($Job_id > 0 or $Verbose_level >= 1) {
            oarsub_print("OAR_JOB_ID=$Job_id\n");
        }
        if (defined(OAR::Sub::signal_almighty($remote_host, $remote_port, "Qsub"))) {
            oarsub_warn(
                "# Error: cannot connect to executor $remote_host:$remote_port. Is OAR started?\n");
            exit(3);
        }
        exit(0);
    } else {
        if ($err == -1) {
            oarsub_warn("# Error: an interactive job or a reservation cannot be resubmitted.\n");
        } elsif ($err == -2) {
            oarsub_warn("# Error: the job must be in Error or Terminated state.\n");
        } elsif ($err == -3) {
            oarsub_warn("# Error: you are not the right user.\n");
        } elsif ($err == -4) {
            oarsub_warn("# Error: another active job is using the same ssh keys.\n");
        } else {
            oarsub_warn("# Error: unknown error ($err).\n");
        }
        exit(4);
    }
}

if ((@ARGV != 1) &&
    ($Interactive == 0)    &&
    ($Reservation eq "0")  &&
    !defined($connect_job) &&
    !grep(/^noop(?:=standby)?$/, @Type)) {
    usage();
    OAR::Sub::close_db_connection();
    exit(5);
}

if (($Interactive == 1) and ($Reservation ne "0")) {
    oarsub_warn("# Error: a reservation cannot be interactive.\n");
    usage();
    OAR::Sub::close_db_connection();
    exit(7);
}

if (($Interactive == 1) and grep(/^noop(?:=standby)?$/, @Type)) {
    oarsub_warn("# Error: a noop job cannot be interactive.\n");
    OAR::Sub::close_db_connection();
    exit(17);
}

if (defined($notify) && $notify =~ m/^.*exec\s*:.+$/m) {
    my $notify_exec_regexp = '[a-zA-Z0-9_.\/ -]+';
    unless ($notify =~ m/.*exec\s*:($notify_exec_regexp)$/m) {
        oarsub_warn(
            "# Error: insecure characters found in notification method (the allowed regexp is: $notify_exec_regexp).\n"
        );
        OAR::Sub::close_db_connection();
        exit(16);
    }
}

# Connect to a reservation
if (defined($connect_job)) {
    if ($connect_job eq 0) {
        my @running_jobs = OAR::Sub::get_running_jobs_for_user($lusr);

        if ($#running_jobs < 0) {
            oarsub_print("No running jobs\n");
            exit(0);
        } elsif ($#running_jobs eq 0) {
            $connect_job = $running_jobs[0]->{job_id};
        } else {
            my $running_jobs_str = join(
                ", ",
                map {
                    "$_->{job_id}" . (defined($_->{job_name}) ? " (" . $_->{job_name} . ")" : "")
                } @running_jobs);
            oarsub_print("You have several running jobs. Please provide a job id.\n");
            oarsub_print("Running jobs are: $running_jobs_str\n");
            exit(0);
        }
    }

    # Do not kill the job if the user close the window
    $SIG{HUP} = 'DEFAULT';
    OAR::Sub::close_db_connection();
    exit(connect_job($connect_job, 0, $Openssh_cmd));
}

# End connection to a reservation

if (($Interactive == 0) and ($ARGV[0] ne "" or grep(/^noop(?:=standby)?$/, @Type))) {
    my $exec = $ARGV[0];
    if (defined($Scan_script)) {
        my $scan_result_tmp = OAR::Sub::scan_script($exec, $Initial_request_string)
          if ($exec ne "");
        my %scan_result = %$scan_result_tmp;
        $Initial_request_string = $scan_result{initial_request};
        if (defined($scan_result{queue}) && !defined($Queue_name)) {
            $Queue_name = $scan_result{queue};
        } elsif (defined($scan_result{queue}) && defined($Queue_name)) {
            oarsub_warn(
                "# Warning: ignore script value for queue parameter: $scan_result{queue}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{property}) && ($Job_sql_properties eq "")) {
            $Job_sql_properties = $scan_result{property};
        } elsif (defined($scan_result{property}) && ($Job_sql_properties ne "")) {
            oarsub_warn(
                "# Warning: ignore script value for property parameter: $scan_result{property}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{resources})) {
            push(@resource, @{ $scan_result{resources} });
        }
        if (defined($scan_result{types})) {
            push(@Type, @{ $scan_result{types} });
        }
        if (defined($scan_result{anterior})) {
            push(@Anterior_job, @{ $scan_result{anterior} });
        }
        if (defined($scan_result{checkpoint}) && ($Checkpoint == 0)) {
            $Checkpoint = $scan_result{checkpoint};
        } elsif (defined($scan_result{checkpoint}) && ($Checkpoint != 0)) {
            oarsub_warn(
                "# Warning: ignore script value for checkpoint parameter: $scan_result{checkpoint}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{notify}) && (!defined($notify))) {
            $notify = $scan_result{notify};
        } elsif (defined($scan_result{notify}) && (defined($notify))) {
            oarsub_warn(
                "# Warning: ignore script value for notify parameter: $scan_result{notify}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{directory})) {
            $Directory = $scan_result{directory};
        }
        if (defined($scan_result{directory}) && (!defined($Directory))) {
            $Directory = $scan_result{directory};
        } elsif (defined($scan_result{directory}) && (defined($Directory))) {
            oarsub_warn(
                "# Warning: ignore script value for name parameter: $scan_result{directory}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{name}) && (!defined($job_name))) {
            $job_name = $scan_result{name};
        } elsif (defined($scan_result{name}) && (defined($job_name))) {
            oarsub_warn(
                "# Warning: ignore script value for name parameter: $scan_result{name}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{project}) && (!defined($Project))) {
            $Project = $scan_result{project};
        } elsif (defined($scan_result{project}) && (defined($Project))) {
            oarsub_warn(
                "# Warning: ignore script value for name parameter: $scan_result{project}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{hold}) && (!defined($job_hold))) {
            $job_hold = $scan_result{hold};
        } elsif (defined($scan_result{hold}) && (defined($job_hold))) {
            oarsub_warn(
                "# Warning: ignore script value for hold parameter: $scan_result{hold}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{signal}) && (!defined($Checkpoint_signal))) {
            $Checkpoint_signal = $scan_result{signal};
        } elsif (defined($scan_result{signal}) && (defined($Checkpoint_signal))) {
            oarsub_warn(
                "# Warning: ignore script value for hold parameter: $scan_result{signal}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{stdout}) && (!defined($Stdout_file))) {
            $Stdout_file = $scan_result{stdout};
        } elsif (defined($scan_result{stdout}) && (defined($Stdout_file))) {
            oarsub_warn(
                "# Warning: ignore script value for stdout parameter: $scan_result{stdout}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{stderr}) && (!defined($Stderr_file))) {
            $Stderr_file = $scan_result{stderr};
        } elsif (defined($scan_result{stderr}) && (defined($Stderr_file))) {
            oarsub_warn(
                "# Warning: ignore script value for stderr parameter: $scan_result{stderr}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{usejobkey})) {
            $use_job_key = 1;
        }
        if (defined($scan_result{importjobkeyinlinepriv}) && ($import_job_key_inline eq "")) {
            $import_job_key_inline = $scan_result{importjobkeyinlinepriv};
        } elsif (defined($scan_result{importjobkeyinlinepriv}) && ($import_job_key_inline ne "")) {
            oarsub_warn(
                "# Warning: ignore script value for import-job-key-inline-priv parameter: $scan_result{importjobkeyinlinepriv}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{importjobkeyfromfile}) && ($import_job_key_file eq "")) {
            $import_job_key_file = $scan_result{importjobkeyfromfile};
        } elsif (defined($scan_result{importjobkeyfromfile}) && ($import_job_key_file ne "")) {
            oarsub_warn(
                "# Warning: ignore script value for import-job-key-from-file parameter: $scan_result{importjobkeyfromfile}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{exportjobkeytofile}) && ($export_job_key_file eq "")) {
            $export_job_key_file = $scan_result{exportjobkeytofile};
        } elsif (defined($scan_result{exportjobkeytofile}) && ($export_job_key_file ne "")) {
            oarsub_warn(
                "# Warning: ignore script value for export-job-key-to-file parameter: $scan_result{exportjobkeytofile}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{array}) && (!defined($array_job_nb))) {
            $array_job_nb = $scan_result{array};
        } elsif (defined($scan_result{array}) && (defined($array_job_nb))) {
            oarsub_warn(
                "# Warning: ignore script value for array parameter: $scan_result{array}; another value was given on the command line.\n"
            );
        }
        if (defined($scan_result{arrayparamfile}) && (!defined($array_param_file))) {
            $array_param_file = $scan_result{arrayparamfile};
        } elsif (defined($scan_result{arrayparamfile}) && (defined($array_param_file))) {
            oarsub_warn(
                "# Warning: ignore script value for array-param-file parameter: $scan_result{arrayparamfile}; another value was given on the command line.\n"
            );
        }
    }

    my @resource_list = parse_resource_descriptions(\@resource);

    $Project           = $Project_default           if (!defined($Project));
    $Checkpoint_signal = $Checkpoint_signal_default if (!defined($Checkpoint_signal));
    $Directory         = $Directory_default         if (!defined($Directory));

    # array_ids are assigned automatically
    if (defined($array_param_file)) {
        $array_params_ref = OAR::Sub::read_array_param_file($array_param_file);
        $array_job_nb     = scalar @{$array_params_ref};
    }

    $array_job_nb = 1 if (!defined($array_job_nb));
    if ($array_job_nb <= 0) {
        oarsub_warn("# Error: an array job must have a positive number of sub-jobs.\n");
        usage();
        OAR::Sub::close_db_connection();
        exit(6);
    }

    $Cmd_executor = "Qsub";

    ($Server, my $server_port) = parse_reservation(0, \@resource_list);

    $Job_id_list_ref = OAR::Sub::add_micheline_job(
        "PASSIVE",               \@resource_list,
        $exec,                   "$Host:$server_port",
        $Queue_name,             $Job_sql_properties,
        $Reservation_start,      defined($idFile) ? $idFile : "NULL",
        $Checkpoint,             $Checkpoint_signal,
        $notify,                 $job_name,
        $job_env,                \@Type,
        $Directory,              \@Anterior_job,
        $Stdout_file,            $Stderr_file,
        $job_hold,               $Project,
        $use_job_key,            $import_job_key_inline,
        $import_job_key_file,    $export_job_key_file,
        $Initial_request_string, $array_job_nb,
        $array_params_ref,       $Verbose_level,
        $micheline_print_func,   $micheline_warn_func,
        $micheline_capture_func);
} else {
    if ($ARGV[0] ne "") {
        oarsub_warn(
            "# Warning: you asked for an Interactive job SO I will ignore arguments: $ARGV[0] ; Is your syntax right?\n"
        );
    }
    $Cmd_executor = "Qsub -I";

    my @resource_list = parse_resource_descriptions(\@resource);

    if (defined($array_param_file)) {
        oarsub_warn("# Error: a parameter file-based array job cannot be interactive.\n");

        # user defined array_ids, 0 if interactive and/or non-array job
        usage();
        OAR::Sub::close_db_connection();
        exit(9);
    }

    $Project           = $Project_default           if (!defined($Project));
    $Checkpoint_signal = $Checkpoint_signal_default if (!defined($Checkpoint_signal));
    $Directory         = $Directory_default         if (!defined($Directory));

    $array_job_nb = 1 if (!defined($array_job_nb));
    if ($array_job_nb != 1) {
        oarsub_warn("# Error: an array job cannot be interactive.\n");
        usage();
        OAR::Sub::close_db_connection();
        exit(8);
    }

    ($Server, my $server_port) = parse_reservation(1, \@resource_list);

    $Job_id_list_ref = OAR::Sub::add_micheline_job(
        "INTERACTIVE",           \@resource_list,
        "",                      "$Host:$server_port",
        $Queue_name,             $Job_sql_properties,
        $Reservation_start,      defined($idFile) ? $idFile : "NULL",
        $Checkpoint,             $Checkpoint_signal,
        $notify,                 $job_name,
        $job_env,                \@Type,
        $Directory,              \@Anterior_job,
        $Stdout_file,            $Stderr_file,
        $job_hold,               $Project,
        $use_job_key,            $import_job_key_inline,
        $import_job_key_file,    $export_job_key_file,
        $Initial_request_string, $array_job_nb,
        $array_params_ref,       $Verbose_level,
        $micheline_print_func,   $micheline_warn_func,
        $micheline_capture_func);
}

if ((!defined($Job_id_list_ref)) or (ref($Job_id_list_ref) ne "ARRAY")) {
    OAR::Sub::close_db_connection();
    exit($Job_id_list_ref);
}

if (@{$Job_id_list_ref} > 1) {
    for my $j (@{$Job_id_list_ref}) {
        if ($j > 0 or $Verbose_level >= 1) {
            oarsub_print("OAR_JOB_ID=" . $j . "\n");
        }
    }
    oarsub_print("OAR_ARRAY_ID=" . OAR::Sub::get_job_array_id($Job_id_list_ref->[0]) . "\n");
} else {
    if ($Job_id_list_ref->[0] > 0 or $Verbose_level >= 1) {
        oarsub_print("OAR_JOB_ID=" . $Job_id_list_ref->[0] . "\n");
    }
}

if ((defined($DUMPER_mode)) or (defined($YAML_mode) or ($XML_mode) or ($JSON_mode))) {
    foreach my $Job_id (@{$Job_id_list_ref}) {
        my $tmp = { job_id => $Job_id };
        if (defined($DUMPER_mode)) {
            print(Dumper($tmp));
        } elsif (defined($XML_mode)) {
            if ($XML_enabled == 1) {
                my $dump = new XML::Dumper;
                $dump->dtd;
                print($dump->pl2xml($tmp));
            } else {
                warn(
                    "XML module not available on the system. Ask your administrator to install it if needed.\n"
                );
            }
        } elsif (defined($YAML_mode)) {
            if ($YAML_enabled == 1) {
                print(YAML::Dump($tmp));
            } else {
                warn(
                    "YAML module not available on the system. Ask your administrator to install it if needed.\n"
                );
            }
        } elsif (defined($JSON_mode)) {
            if ($JSON_enabled == 1) {
                print(JSON->new->pretty(1)->encode($tmp));
            } else {
                warn(
                    "JSON module not available on the system. Ask your administrator to install it if needed.\n"
                );
            }
        }
    }
}

OAR::Sub::close_db_connection();

if ((@{$Job_id_list_ref} < 1) or ($Job_id_list_ref->[-1] <= 0)) {
    oarsub_warn("# Error: please verify your job submission syntax.\n");
    exit(8);
}

#Signal Almigthy
if (defined(OAR::Sub::signal_almighty($remote_host, $remote_port, "$Cmd_executor"))) {

    #qdel(1);
    oarsub_warn("# Error: cannot connect to executor $remote_host:$remote_port. Job is killed.\n");
    exit(9);
}

my $answer;
if ($Reservation ne "0") {

    #Reservation mode
    oarsub_print("# Advance reservation request: waiting for validation...\n");
    my $client = $Server->accept();
    my $advance_reservation_validation_hook =
      get_conf("ADVANCE_RESERVATION_VALIDATION_HOOK_EXEC_FILE");
    $answer = <$client>;
    chop($answer);
    if ($answer eq "GOOD RESERVATION") {
        oarsub_print("# Reservation valid --> OK\n");
        if (defined($advance_reservation_validation_hook)) {
            run_advance_reservation_validation_hook($advance_reservation_validation_hook, 1);
        }
    } else {
        oarsub_print("# Reservation not valid --> KO ($answer)\n");
        if (defined($advance_reservation_validation_hook)) {
            run_advance_reservation_validation_hook($advance_reservation_validation_hook, 0);
        }
        exit(10);
    }
} elsif ($Interactive == 1) {

    #Interactive mode
    oarsub_print("# Interactive mode: waiting...\n");
    my $prev_str = "";
    do {
        my $client = $Server->accept();
        $answer = <$client>;
        chomp($answer);
        if ($answer =~ /\](.*)$/) {
            if ($1 ne $prev_str) {
                oarsub_print("# $answer\n");
                $prev_str = $1;
            }
        } elsif ($answer ne "GOOD JOB") {
            oarsub_print("# $answer\n");
        }
      } while (($answer ne "GOOD JOB") and
        ($answer ne "BAD JOB")    and
        ($answer ne "JOB KILLED") and
        ($answer !~ /^ERROR/));
    if ($answer eq "GOOD JOB") {
        exit(connect_job($Job_id_list_ref->[0], 1, $Openssh_cmd));
    } else {
        exit(11);
    }
}

exit(0);
