diff --git a/.gitignore b/.gitignore
index 65476f3bad3e8678e256558f671bba589eafa437..7589a8fe8d1f9c216aa281abbca47c9b96d4c869 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,9 @@ Makefile
/etc/cgroup.release_common.example
/etc/init.d.slurm
/etc/init.d.slurmdbd
+/etc/slurmctld.service
+/etc/slurmd.service
+/etc/slurmdbd.service
/libtool
/slurm/slurm.h
/slurm/stamp-h2
diff --git a/META b/META
index 9872fa58c37d2250a6518ba5d293027875a4ebb4..58972f13aaeca3803f133c340dd6b4e3f8b53171 100644
--- a/META
+++ b/META
@@ -9,8 +9,8 @@
Name: slurm
Major: 15
Minor: 08
- Micro: 2
- Version: 15.08.2
+ Micro: 3
+ Version: 15.08.3
Release: 1
##
diff --git a/NEWS b/NEWS
index 868e82166fe3b4e539a9aaf55b4724025ae16cff..090ea4bb2ee9906e6709ca4556e95f004055edb9 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,53 @@
This file describes changes in recent versions of Slurm. It primarily
documents those changes that are of interest to users and administrators.
+* Changes in Slurm 15.08.3
+==========================
+ -- Correct Slurm's RPM build if Munge is not installed.
+ -- Job array termination status email ExitCode based upon highest exit code
+ from any task in the job array rather than the last task. Also change the
+ state from "Ended" or "Failed" to "Mixed" where appropriate.
+ -- Squeue recombines pending job array records only if their name and partition
+ are identical.
+ -- Fix some minor leaks in the job info and step info API.
+ -- Export missing QOS id when filling in association with the association
+ manager.
+ -- Fix invalid reference if a lua job_submit plugin references a default qos
+ when a user doesn't exist in the database.
+ -- Use association enforcement in the lua plugin.
+ -- Fix a few spots missing defines of accounting_enforce or acct_db_conn
+ in the plugins.
+ -- Show requested TRES in scontrol show jobs when job is pending.
+ -- Improve sched/backfill support for job features, especially XOR construct.
+ -- Correct scheduling logic for job features option with XOR construct that
+ could delay a job's initiation.
+ -- Remove unneeded frees when creating a tres string.
+ -- Send a tres_alloc_str for the batch step
+ -- Fix incorrect check for slurmdb_find_tres_count_in_string in various places,
+ it needed to check for INFINITE64 instead of zero.
+ -- Don't allow scontrol to create partitions with the name "DEFAULT".
+ -- burst_buffer/cray: Change error from "invalid request" to "permssion denied"
+ if a non-authorized user tries to create/destroy a persistent buffer.
+ -- PrologFlags work: Setting a flag of "Contain" implicitly sets the "Alloc"
+ flag. Fix code path which could prevent execution of the Prolog when the
+ "Alloc" or "Contain" flag were set.
+ -- Fix for acct_gather_energy/cray|ibmaem to work with missed enum.
+ -- MYSQL - When inserting a job and begin_time is 0 do not set it to
+ submit_time. 0 means the job isn't eligible yet so we need to treat it so.
+ -- MYSQL - Don't display ineligible jobs when querying for a window of time.
+ -- Fix creation of advanced reservation of cores on nodes which are DOWN.
+ -- Return permission denied if regular user tries to release job held by an
+ administrator.
+ -- MYSQL - Fix rollups for multiple jobs running by the same association
+ in an hour counting multiple times.
+ -- Burstbuffer/Cray plugin - Fix for persistent burst buffer use.
+ Don't call paths if no #DW options.
+ -- Modifications to pam_slurm_adopt to work correctly for the "extern" step.
+ -- Alphabetize debugflags when printing them out.
+ -- Fix systemd's slurmd service from killing slurmstepds on shutdown.
+ -- Fixed counter of not indexed jobs, error_cnt post-increment changed to
+ pre-increment.
+
* Changes in Slurm 15.08.2
==========================
-- Fix for tracking node state when jobs that have been allocated exclusive
@@ -499,6 +546,10 @@ documents those changes that are of interest to users and administrators.
has been modified to accept a "Pack" and "NoPack" option. These options can
be used to override the CR_PACK_NODE configuration option.
+* Changes in Slurm 14.11.11
+===========================
+ -- Fix systemd's slurmd service from killing slurmstepds on shutdown.
+
* Changes in Slurm 14.11.10
===========================
-- Fix truncation of job reason in squeue.
@@ -544,6 +595,8 @@ documents those changes that are of interest to users and administrators.
-- Fix issue with sacct, printing 0_0 for array's that had finished in the
database but the start record hadn't made it yet.
-- Fix sacct -j, (nothing but a comma) to not return all jobs.
+ -- Prevent slurmstepd from core dumping if /proc/<pid>/stat has
+ unexpected format.
* Changes in Slurm 14.11.9
==========================
diff --git a/contribs/pam_slurm_adopt/README b/contribs/pam_slurm_adopt/README
index c4658db313d9e3e5dcc23926fcc0d11469257a0d..15799630ed0f13ebdd1857948b790934f055c4e3 100644
--- a/contribs/pam_slurm_adopt/README
+++ b/contribs/pam_slurm_adopt/README
@@ -14,7 +14,7 @@ DESCRIPTION
This module attempts to determine the job which originated this connection.
The module is configurable; these are the default steps:
- 1) Check the local stepds for a count of jobs owned by the non-root user
+ 1) Check the local stepd for a count of jobs owned by the non-root user
a) If none, deny (option action_no_jobs)
b) If only one, adopt the process into that job
c) If multiple, continue
@@ -38,36 +38,72 @@ This module has the following options (* = default):
a service or similar, it will be tracked and killed by Slurm
when the job exits. This sounds bad because it is bad.
- 1* = let the connection through without adoption
+ 1* = Let the connection through without adoption
0 = I am crazy. I want random services to die when root jobs exit. I
also like it when RPC calls block for a while then time out.
action_no_jobs - The action to perform if the user has no jobs on the node
- ignore = let the connection through without adoption
- deny* = deny the connection
+ ignore = Do nothing. Fall through to the next pam module
+ deny* = Deny the connection
- action_unknown - The action to perform when the RPC call does not locate the
- source job and the user has multiple jobs on the node to
- choose from
+ action_unknown - The action to perform when the user has multiple jobs on
+ the node *and* the RPC call does not locate the source job.
+ If the RPC mechanism works properly in your environment,
+ this option will likely be relevant *only* when connecting
+ from a login node.
- any* = pick a job in a (somewhat) random fashion
- ignore = let the connection through without adoption
- deny = deny the connection
+ newest* = Pick the newest job on the node. The "newest" job is chosen
+ based on the mtime of the job's step_extern cgroup; asking
+ Slurm would require an RPC to the controller. The user can ssh
+ in but may be adopted into a job that exits earlier than the
+ job they intended to check on. The ssh connection will at
+ least be subject to appropriate limits and the user can be
+ informed of better ways to accomplish their objectives if this
+ becomes a problem
+ user = Use the /slurm/uid_$UID cgroups. Not all cgroups set
+ appropriate limits at this level so this may not be very
+ effective. Additionally, job accounting at this level is
+ impossible as is automatic cleanup of stray processes when the
+ job exits. This settings is not recommended.
+ allow = Let the connection through without adoption
+ deny = Deny the connection
- action_adopt_failure - The action to perform if the job is unable to be
- adopted into a job for whatever reason
+ action_adopt_failure - The action to perform if the process is unable to be
+ adopted into an identified job for whatever reason
- ignore = let the connection through without adoption
- deny* = deny the connection
+ allow* = Let the connection through without adoption
+ deny = Deny the connection
+ action_generic_failure - The action to perform it there certain failures
+ such as inability to talk to the local slurmd or
+ if the kernel doesn't offer the correct facilities
+
+ ignore* = Do nothing. Fall through to the next pam module
+ allow = Let the connection through without adoption
+ deny = Deny the connection
log_level - See SlurmdDebug in slurm.conf(5) for available options. The
default log_level is info.
+SLURM.CONF CONFIGURATION
+ For best results, all relevant cgroups plugins (e.g. proctrack/cgroup) should
+ be enabled in slurm.conf. At least one must be enabled for this module to be
+ even somewhat useful.
+
+ PrologFlags=contain must be set in slurm.conf. This sets up the "extern" step
+ into which ssh-launched processes will be adopted.
+
+ **** IMPORTANT ****
+ PrologFlags=contain must be in place *before* using this module.
+ The module bases its checks on local steps that have already been launched. If
+ the user has no steps on the node, such as the extern step, the module will
+ assume that the user has no jobs allocated to the node. Depending on your
+ configuration of the pam module, you might deny *all* user ssh attempts.
+
NOTES
This module and the related RPC call currently support Linux systems which
have network connection information available through /proc/net/tcp{,6}. A
@@ -79,20 +115,21 @@ NOTES
Slurm is tracking.
IPv6 is supported by the RPC data structure itself and the code which sends it
- or receives it. Sending the RPC call to an IPv6 address is not currently
+ and receives it. Sending the RPC call to an IPv6 address is not currently
supported by Slurm. Once support is added, remove the relevant check in
- slurm_network_callerid ().
+ slurm_network_callerid().
- proctrack/cgroup is recommended on Linux.
+ One future action_unknown idea is an option to pick the job with the longest
+ time remaining. This is not yet implemented.
FIREWALLS, IP ADDRESSES, ETC.
- slurmd should be accessible on any IP address that a user might launch ssh.
- The RPC call to determine the source job must be able to reach the slurmd port
- on that particular IP address.
+ slurmd should be accessible on any IP address from which a user might launch
+ ssh. The RPC call to determine the source job must be able to reach the slurmd
+ port on that particular IP address.
- If there is no slurmd on the source node, it is better to have the RPC call be
- rejected rather than silently dropped. This will allow better responsiveness
- to the RPC initiator.
+ If there is no slurmd on the source node, such as on a login node, it is
+ better to have the RPC call be rejected rather than silently dropped. This
+ will allow better responsiveness to the RPC initiator.
EXAMPLES / SUGGESTED USAGE
Use of this module is recommended on any compute node.
@@ -100,10 +137,12 @@ EXAMPLES / SUGGESTED USAGE
Add the following line to the appropriate file in /etc/pam.d, such as
system-auth or sshd:
- account required pam_slurm_adopt.so
+ account sufficient pam_slurm_adopt.so
- If you always want to allow access for an administrative group (eg, wheel),
- stack the pam_access module ahead of pam_slurm:
+ If you always want to allow access for an administrative group (e.g. wheel),
+ stack the pam_access module after pam_slurm_adopt. A success with
+ pam_slurm_adopt is sufficient to allow access but the pam_access module can
+ allow others, such as staff, access even without jobs.
account sufficient pam_slurm_adopt.so
account required pam_access.so
diff --git a/contribs/pam_slurm_adopt/pam_slurm_adopt.c b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
index 8af62b7ce4ddebcc68c73bba85811c3e1b5540bc..100003c25f16c69598cd00dfe8866bd8cf3aa07a 100644
--- a/contribs/pam_slurm_adopt/pam_slurm_adopt.c
+++ b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
@@ -65,6 +65,8 @@
#include "slurm/slurm.h"
#include "src/common/slurm_xlator.h"
#include "src/common/slurm_protocol_api.h"
+#include "src/common/xcgroup_read_config.c"
+#include "src/slurmd/common/xcgroup.c"
/* This definition would probably be good to centralize somewhere */
#ifndef MAXHOSTNAMELEN
@@ -72,16 +74,21 @@
#endif
typedef enum {
- CALLERID_ACTION_ANY,
+ CALLERID_ACTION_NEWEST,
+ CALLERID_ACTION_ALLOW,
CALLERID_ACTION_IGNORE,
- CALLERID_ACTION_DENY
+ CALLERID_ACTION_DENY,
+ CALLERID_ACTION_USER
} callerid_action_t;
/* module options */
static struct {
- int single_job_skip_rpc;
+ int single_job_skip_rpc; /* Undocumented. If 1 and there is only 1 user
+ * job, adopt it and skip RPC. If 0, *always*
+ * try RPC even in single job situations.
+ * Unlikely to ever be set to 0. */
int ignore_root;
- int action_no_jobs;
+ callerid_action_t action_no_jobs;
callerid_action_t action_unknown;
callerid_action_t action_adopt_failure;
callerid_action_t action_generic_failure;
@@ -93,27 +100,85 @@ static void _init_opts(void)
opts.single_job_skip_rpc = 1;
opts.ignore_root = 1;
opts.action_no_jobs = CALLERID_ACTION_DENY;
- opts.action_unknown = CALLERID_ACTION_ANY;
- opts.action_adopt_failure = CALLERID_ACTION_IGNORE;
- opts.action_generic_failure = CALLERID_ACTION_IGNORE;
+ opts.action_unknown = CALLERID_ACTION_NEWEST;
+ opts.action_adopt_failure = CALLERID_ACTION_ALLOW;
+ opts.action_generic_failure = CALLERID_ACTION_ALLOW;
opts.log_level = LOG_LEVEL_INFO;
}
-static int _adopt_process(pid_t pid, uint32_t job_id)
+/* Returns SLURM_SUCCESS if opts.action_adopt_failure == CALLERID_ACTION_ALLOW
+ * or if the process can be adopted into any cgroup. The admin might not have
+ * configured all the cgroups, so we'll assume that a success in one means they
+ * were adopted into all the configured ones. A TODO item is to only attempt
+ * adoption into the configured cgroups.
+ *
+ * If job_id==NO_VAL, the process will be adopted into the uid_%u cgroups only.
+ */
+static int _adopt_process(pid_t pid, uint32_t job_id, uid_t uid)
{
- /* TODO: add this pid to plugins for task, container, accounting, etc
- * need more code here ... */
- info("_adopt_process(%d, %u): UNIMPLEMENTED", pid, job_id);
+ xcgroup_t cg;
+ xcgroup_ns_t ns;
+ int i, rc, cgroup_type_count = 5;
+ char *cgroup_types[] =
+ { "memory", "cpuset", "cpuacct", "freezer", "device" };
+ char path[PATH_MAX];
+
+ /* Set default return code based on settings */
+ rc = opts.action_adopt_failure == CALLERID_ACTION_ALLOW ?
+ PAM_SUCCESS : PAM_PERM_DENIED;
+
+ debug3("Calling _adopt_process(%d, %u, %u)", pid, job_id, uid);
+
+ /* job_id == NO_VAL indicates that we should use the uid_%s cgroup */
+ if (job_id == NO_VAL)
+ snprintf(path, PATH_MAX, "/slurm/uid_%u", uid);
+ else
+ snprintf(path, PATH_MAX, "/slurm/uid_%u/job_%d/step_extern",
+ uid, job_id);
+
+ for (i = 0; i < cgroup_type_count; i++) {
+ if (xcgroup_ns_load(slurm_cgroup_conf, &ns, cgroup_types[i])
+ != SLURM_SUCCESS) {
+ info("_adopt_process(%d, %u, %u): xcgroup_ns_load failed for %s",
+ pid, job_id, uid, cgroup_types[i]);
+ continue;
+ }
+ if (xcgroup_load(&ns, &cg, path) != SLURM_SUCCESS) {
+ info("_adopt_process(%d, %u, %u): xcgroup_load failed for cgroup %s, path %s",
+ pid, job_id, uid, cgroup_types[i], path);
+ continue;
+ }
+ if (xcgroup_set_uint64_param(&cg, "tasks", (uint64_t)pid)
+ != SLURM_SUCCESS) {
+ info("_adopt_process(%d, %u, %u): adding pid %d to %s/tasks failed",
+ pid, job_id, uid, pid, cg.path);
+ continue;
+ }
+ debug("_adopt_process(%d, %u, %u): pid %d adopted into %s",
+ pid, job_id, uid, pid, cg.path);
+ /* We will consider one success to be good enough */
+ rc = PAM_SUCCESS;
+ }
- /* TODO: change my primary gid to the job's group, if possible */
- return SLURM_SUCCESS;
+ if (rc == PAM_SUCCESS)
+ info("Process %d adopted into job %u", pid, job_id);
+ else
+ info("Process %d adoption FAILED for all cgroups of job %u",
+ pid, job_id);
+
+ /* TODO: Change my primary gid to the job's group after
+ * https://bugzilla.mindrot.org/show_bug.cgi?id=2380 is merged.
+ * If you are reading this message and nothing has been done with
+ * that bug, please consider adding a "I would like this too"
+ * comment.
+ */
+ return rc;
}
/* Returns negative number on failure. Failures are likely to occur if a step
* exits; this is not a problem. */
static uid_t _get_job_uid(step_loc_t *stepd)
{
- /* BUG: uid_t on Linux is unsigned but stepd_get_uid can return -1 */
uid_t uid = -1;
int fd;
uint16_t protocol_version;
@@ -131,7 +196,6 @@ static uid_t _get_job_uid(step_loc_t *stepd)
close(fd);
/* The step may have exited. Not a big concern. */
- /* BUG: uid_t on Linux is unsigned but stepd_get_uid can return -1 */
if ((int32_t)uid == -1)
debug3("unable to determine uid of step %u.%u on %s",
stepd->jobid, stepd->stepid, stepd->nodename);
@@ -139,12 +203,35 @@ static uid_t _get_job_uid(step_loc_t *stepd)
return uid;
}
+/* Return mtime of a cgroup. If we can't read the right cgroup information,
+ * return 0. That results in a (somewhat) random choice of job */
+static time_t _cgroup_creation_time(char *uidcg, uint32_t job_id)
+{
+ char path[PATH_MAX];
+ struct stat statbuf;
+
+ if (snprintf(path, PATH_MAX, "%s/job_%u", uidcg, job_id) >= PATH_MAX) {
+ info("snprintf: '%s/job_%u' longer than PATH_MAX of %d",
+ uidcg, job_id, PATH_MAX);
+ return 0;
+ }
+
+ if (stat(path, &statbuf) != 0) {
+ info("Couldn't stat path '%s'", path);
+ return 0;
+ }
+
+ return statbuf.st_mtime;
+}
+
static int _indeterminate_multiple(pam_handle_t *pamh, List steps, uid_t uid,
uint32_t *job_id)
{
ListIterator itr = NULL;
- int rc = SLURM_FAILURE;
+ int rc = PAM_PERM_DENIED;
step_loc_t *stepd = NULL;
+ time_t most_recent = 0, cgroup_time = 0;
+ char uidcg[PATH_MAX];
if (opts.action_unknown == CALLERID_ACTION_DENY) {
debug("Denying due to action_unknown=deny");
@@ -153,22 +240,43 @@ static int _indeterminate_multiple(pam_handle_t *pamh, List steps, uid_t uid,
PAM_MODULE_NAME
": unable to determine source job");
return PAM_PERM_DENIED;
- } else if (opts.action_unknown == CALLERID_ACTION_IGNORE) {
- debug("Allowing due to action_unknown=ignore");
+ } else if (opts.action_unknown == CALLERID_ACTION_USER) {
+ debug("Using uid_%u cgroups due to action_unknown=user", uid);
+ *job_id = (uint32_t)NO_VAL;
return PAM_SUCCESS;
}
+ if (snprintf(uidcg, PATH_MAX, "%s/memory/slurm/uid_%u",
+ slurm_cgroup_conf->cgroup_mountpoint, uid) >= PATH_MAX) {
+ info("snprintf: '%s/memory/slurm/uid_%u' longer than PATH_MAX of %d",
+ slurm_cgroup_conf->cgroup_mountpoint, uid, PATH_MAX);
+ /* Make the uidcg an empty string. This will effectively switch
+ * to a (somewhat) random selection of job rather than picking
+ * the latest, but how did you overflow PATH_MAX chars anyway?
+ */
+ uidcg[0] = '\0';
+ }
+
itr = list_iterator_create(steps);
while ((stepd = list_next(itr))) {
- if (uid == _get_job_uid(stepd)) {
- *job_id = stepd->jobid;
- rc = SLURM_SUCCESS;
- break;
+ /* Only use container steps from this user */
+ if (stepd->stepid == SLURM_EXTERN_CONT &&
+ uid == _get_job_uid(stepd)) {
+ cgroup_time = _cgroup_creation_time(
+ uidcg, stepd->jobid);
+ /* Return the newest job_id, according to cgroup
+ * creation. Hopefully this is a good way to do this */
+ if (cgroup_time > most_recent) {
+ most_recent = cgroup_time;
+ *job_id = stepd->jobid;
+ rc = PAM_SUCCESS;
+ }
}
}
- /* No jobs from this user exist on this node */
- if (rc != SLURM_SUCCESS) {
+ /* No jobs from this user exist on this node. This should have been
+ * caught earlier but wasn't for some reason. */
+ if (rc != PAM_SUCCESS) {
if (opts.action_no_jobs == CALLERID_ACTION_DENY) {
debug("uid %u owns no jobs => deny", uid);
send_user_msg(pamh,
@@ -177,9 +285,9 @@ static int _indeterminate_multiple(pam_handle_t *pamh, List steps, uid_t uid,
": you have no active jobs on this node");
rc = PAM_PERM_DENIED;
} else {
- debug("uid %u owns no jobs but action_no_jobs=ignore",
+ debug("uid %u owns no jobs but action_no_jobs=allow",
uid);
- rc = PAM_IGNORE;
+ rc = PAM_SUCCESS;
}
}
@@ -187,10 +295,36 @@ static int _indeterminate_multiple(pam_handle_t *pamh, List steps, uid_t uid,
return rc;
}
-static int _single_job_check(List steps, uid_t uid, uint32_t *job_id)
+static int _action_unknown(pam_handle_t *pamh, struct passwd *pwd, List steps)
+{
+ int rc;
+ uint32_t job_id;
+
+ if (opts.action_unknown == CALLERID_ACTION_ALLOW) {
+ debug("Allowing due to action_unknown=allow");
+ return PAM_SUCCESS;
+ }
+
+ /* Both the single job check and the RPC call have failed to ascertain
+ * the correct job to adopt this into. Time for drastic measures */
+ rc = _indeterminate_multiple(pamh, steps, pwd->pw_uid, &job_id);
+ if (rc == PAM_SUCCESS) {
+ info("action_unknown: Picked job %u", job_id);
+ rc = _adopt_process(getpid(), job_id, pwd->pw_uid);
+ } else {
+ /* This pam module was worthless, apparently */
+ debug("_indeterminate_multiple failed to find a job to adopt this into");
+ }
+
+ return rc;
+}
+
+/* _user_job_count returns the count of jobs owned by the user AND sets job_id
+ * to the last job from the user that is found */
+static int _user_job_count(List steps, uid_t uid, uint32_t *job_id)
{
ListIterator itr = NULL;
- int user_job_cnt = 0, rc = SLURM_FAILURE;
+ int user_job_cnt = 0;
step_loc_t *stepd = NULL;
*job_id = (uint32_t)NO_VAL;
@@ -203,19 +337,12 @@ static int _single_job_check(List steps, uid_t uid, uint32_t *job_id)
if (*job_id != stepd->jobid) {
user_job_cnt++;
*job_id = stepd->jobid;
- rc = SLURM_SUCCESS;
}
}
- if(user_job_cnt > 1) {
- debug3("_single_job_check: uid %u has multiple jobs on this node",
- uid);
- rc = SLURM_FAILURE;
- break;
- }
}
list_iterator_destroy(itr);
- return rc;
+ return user_job_cnt;
}
static int _rpc_network_callerid(struct callerid_conn *conn, char *user_name,
@@ -255,6 +382,54 @@ static int _rpc_network_callerid(struct callerid_conn *conn, char *user_name,
}
}
+static int _try_rpc(struct passwd *pwd)
+{
+ uint32_t job_id;
+ int rc;
+ char ip_src_str[INET6_ADDRSTRLEN];
+ struct callerid_conn conn;
+
+ /* Gather network information for RPC call. */
+ debug("Checking file descriptors for network socket");
+
+ /* Check my fds for a network socket */
+ if (callerid_get_own_netinfo(&conn) != SLURM_SUCCESS) {
+ /* If this failed, the RPC will surely fail. If we continued
+ * we'd have to fill in junk for lots of variables. Return so
+ * that action_unknown will happen */
+ error("Unable to find network socket");
+ if (opts.action_generic_failure == CALLERID_ACTION_DENY)
+ return PAM_PERM_DENIED;
+ else
+ return PAM_IGNORE;
+ }
+
+ if (inet_ntop(conn.af, &conn.ip_src, ip_src_str, INET6_ADDRSTRLEN)
+ == NULL) {
+ /* This is really odd. If this failed, other functions are so
+ * likely to fail that we might as well skip the RPC */
+ error("inet_ntop failed");
+ if (opts.action_generic_failure == CALLERID_ACTION_DENY)
+ return PAM_PERM_DENIED;
+ else
+ return PAM_IGNORE;
+ }
+
+ /* Ask the slurmd at the source IP address about this connection */
+ rc = _rpc_network_callerid(&conn, pwd->pw_name, &job_id);
+ if (rc == SLURM_SUCCESS) {
+ rc = _adopt_process(getpid(), job_id, pwd->pw_uid);
+ return rc;
+ }
+
+ info("From %s port %d as %s: unable to determine source job",
+ ip_src_str,
+ conn.port_src,
+ pwd->pw_name);
+
+ return PAM_IGNORE;
+}
+
/* Use the pam logging function for now since normal logging is not yet
* initialized */
log_level_t _parse_log_level(pam_handle_t *pamh, const char *log_level_str)
@@ -314,22 +489,50 @@ static void _parse_opts(pam_handle_t *pamh, int argc, const char **argv)
char *v;
for (; argc-- > 0; ++argv) {
- if (!strncasecmp(*argv, "single_job_skip_rpc=0", 18))
+ if (!strncasecmp(*argv, "single_job_skip_rpc=0", 21))
opts.single_job_skip_rpc = 0;
else if (!strncasecmp(*argv, "ignore_root=0", 13))
opts.ignore_root = 0;
- else if (!strncasecmp(*argv,"action_unknown=",15)) {
+ else if (!strncasecmp(*argv,"action_no_jobs=",15)) {
+ v = (char *)(15 + *argv);
+ if (!strncasecmp(v, "deny", 4))
+ opts.action_no_jobs = CALLERID_ACTION_DENY;
+ else if (!strncasecmp(v, "ignore", 6))
+ opts.action_no_jobs = CALLERID_ACTION_IGNORE;
+ else {
+ pam_syslog(pamh,
+ LOG_ERR,
+ "unrecognized action_no_jobs=%s, setting to 'deny'",
+ v);
+ }
+ } else if (!strncasecmp(*argv,"action_unknown=",15)) {
v = (char *)(15 + *argv);
- if (!strncasecmp(v, "ignore", 6))
- opts.action_unknown = CALLERID_ACTION_IGNORE;
- else if (!strncasecmp(v, "any", 3))
- opts.action_unknown = CALLERID_ACTION_ANY;
+ if (!strncasecmp(v, "allow", 5))
+ opts.action_unknown = CALLERID_ACTION_ALLOW;
+ else if (!strncasecmp(v, "newest", 6))
+ opts.action_unknown = CALLERID_ACTION_NEWEST;
else if (!strncasecmp(v, "deny", 4))
opts.action_unknown = CALLERID_ACTION_DENY;
+ else if (!strncasecmp(v, "user", 4))
+ opts.action_unknown = CALLERID_ACTION_USER;
else {
pam_syslog(pamh,
LOG_ERR,
- "unrecognized action_unknown=%s, setting to 'any'",
+ "unrecognized action_unknown=%s, setting to 'newest'",
+ v);
+ }
+ } else if (!strncasecmp(*argv,"action_generic_failure=",23)) {
+ v = (char *)(23 + *argv);
+ if (!strncasecmp(v, "allow", 5))
+ opts.action_generic_failure = CALLERID_ACTION_ALLOW;
+ else if (!strncasecmp(v, "ignore", 6))
+ opts.action_generic_failure = CALLERID_ACTION_IGNORE;
+ else if (!strncasecmp(v, "deny", 4))
+ opts.action_generic_failure = CALLERID_ACTION_DENY;
+ else {
+ pam_syslog(pamh,
+ LOG_ERR,
+ "unrecognized action_generic_failure=%s, setting to 'allow'",
v);
}
} else if (!strncasecmp(*argv, "log_level=", 10)) {
@@ -349,6 +552,18 @@ static void _log_init(log_level_t level)
log_init(PAM_MODULE_NAME, logopts, LOG_AUTHPRIV, NULL);
}
+static int _load_cgroup_config()
+{
+ slurm_cgroup_conf = xmalloc(sizeof(slurm_cgroup_conf_t));
+ bzero(slurm_cgroup_conf, sizeof(slurm_cgroup_conf_t));
+ if (read_slurm_cgroup_conf(slurm_cgroup_conf) != SLURM_SUCCESS) {
+ info("read_slurm_cgroup_conf failed");
+ return SLURM_FAILURE;
+ }
+ return SLURM_SUCCESS;
+}
+
+
/* Parse arguments, etc then get my socket address/port information. Attempt to
* adopt this process into a job in the following order:
* 1) If the user has only one job on the node, pick that one
@@ -360,20 +575,32 @@ static void _log_init(log_level_t level)
PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
__attribute__((unused)), int argc, const char **argv)
{
- int retval = PAM_IGNORE, rc = PAM_IGNORE;
+ int retval = PAM_IGNORE, rc, bufsize, user_jobs;
char *user_name;
- struct callerid_conn conn;
uint32_t job_id;
- char ip_src_str[INET6_ADDRSTRLEN];
List steps = NULL;
struct passwd pwd, *pwd_result;
char *buf = NULL;
- int bufsize;
_init_opts();
_parse_opts(pamh, argc, argv);
_log_init(opts.log_level);
+ switch (opts.action_generic_failure) {
+ case CALLERID_ACTION_DENY:
+ rc = PAM_PERM_DENIED;
+ break;
+ case CALLERID_ACTION_ALLOW:
+ rc = PAM_SUCCESS;
+ break;
+ case CALLERID_ACTION_IGNORE:
+ rc = PAM_IGNORE;
+ break;
+ /* Newer gcc versions warn if enum cases are missing */
+ default:
+ error("The code is broken!!!!");
+ }
+
retval = pam_get_item(pamh, PAM_USER, (void *) &user_name);
if (user_name == NULL || retval != PAM_SUCCESS) {
pam_syslog(pamh, LOG_ERR, "No username in PAM_USER? Fail!");
@@ -384,9 +611,9 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
* basic check that shouldn't be 100% relied on */
if (!opts.ignore_root &&
(opts.action_unknown == CALLERID_ACTION_DENY ||
- opts.action_no_jobs != CALLERID_ACTION_IGNORE ||
- opts.action_adopt_failure != CALLERID_ACTION_IGNORE ||
- opts.action_generic_failure != CALLERID_ACTION_IGNORE
+ opts.action_no_jobs != CALLERID_ACTION_ALLOW ||
+ opts.action_adopt_failure != CALLERID_ACTION_ALLOW ||
+ opts.action_generic_failure != CALLERID_ACTION_ALLOW
)) {
/* Let's get verbose */
info("===============================");
@@ -398,7 +625,7 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
opts.ignore_root = 1;
}
- /* Ignoring root is probably best but the admin can allow it*/
+ /* Ignoring root is probably best but the admin can allow it */
if (!strcmp(user_name, "root")) {
if (opts.ignore_root) {
info("Ignoring root user");
@@ -428,34 +655,22 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
return PAM_SESSION_ERR;
}
- /* Check my fds for a network socket */
- if (callerid_get_own_netinfo(&conn) != SLURM_SUCCESS) {
- /* We could press on for the purposes of the single- or
- * multi-job checks, but the RPC will surely fail. If we
- * continued we'd have to fill in junk for lots of variables */
- error("Unable to find network socket");
- rc = PAM_IGNORE;
- goto cleanup;
- }
+ if (_load_cgroup_config() != SLURM_SUCCESS)
+ return rc;
- if (inet_ntop(conn.af, &conn.ip_src, ip_src_str, INET6_ADDRSTRLEN)
- == NULL) {
- /* This is really odd. If this failed, other functions are so
- * likely to fail that we might as well exit */
- error("inet_ntop failed");
- rc = PAM_IGNORE;
- goto cleanup;
- }
-
- /* Get a list of steps on the node. A failure here likely means failures
- * everywhere so exit on failure or if no local jobs exist */
+ /* Check if there are any steps on the node from any user. A failure here
+ * likely means failures everywhere so exit on failure or if no local jobs
+ * exist. */
steps = stepd_available(NULL, NULL);
if (!steps) {
- error("Error obtaining local step information. Fail.");
- rc = PAM_IGNORE;
+ error("Error obtaining local step information.");
goto cleanup;
- } else if (list_count(steps) == 0) {
- info("No steps on this node from any user");
+ }
+
+ /* Check to see if this user has only one job on the node. If so, choose
+ * that job and adopt this process into it (unless configured not to) */
+ user_jobs = _user_job_count(steps, pwd.pw_uid, &job_id);
+ if (user_jobs == 0) {
if (opts.action_no_jobs == CALLERID_ACTION_DENY) {
send_user_msg(pamh,
"Access denied by "
@@ -463,70 +678,38 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
": you have no active jobs on this node");
rc = PAM_PERM_DENIED;
} else {
- info("uid %u owns no jobs but action_no_jobs=ignore",
+ debug("uid %u owns no jobs but action_no_jobs=ignore",
pwd.pw_uid);
rc = PAM_IGNORE;
}
-
goto cleanup;
- }
-
- /* Check to see if this user has only one job on the node. If so, choose
- * that job and adopt this process into it (unless configured not to) */
- if (opts.single_job_skip_rpc) {
- if (_single_job_check(steps, pwd.pw_uid, &job_id)
- == SLURM_SUCCESS) {
- debug("From %s port %d as %s: _single_job_check succeeded",
- ip_src_str,
- conn.port_src,
- user_name);
-
- info("From %s port %d as %s: member of job %u",
- ip_src_str,
- conn.port_src,
+ } else if (user_jobs == 1) {
+ if (opts.single_job_skip_rpc) {
+ info("Connection by user %s: user has only one job %u",
user_name,
job_id);
- rc = _adopt_process(getpid(), job_id);
+ rc = _adopt_process(getpid(), job_id, pwd.pw_uid);
goto cleanup;
- } else {
- debug("From %s port %d as %s: _single_job_check failed",
- ip_src_str,
- conn.port_src,
- user_name);
}
+ } else {
+ debug("uid %u has %d jobs", pwd.pw_uid, user_jobs);
}
- /* Single job check failed or wasn't used. Ask the slurmd (if any) at
- * the source IP address about this connection */
- rc = _rpc_network_callerid(&conn, user_name, &job_id);
- if (rc == SLURM_SUCCESS) {
- rc = _adopt_process(getpid(), job_id);
+ /* Single job check turned up nothing (or we skipped it). Make RPC call
+ * to slurmd at source IP. If it can tell us the job, the function calls
+ * _adopt_process */
+ rc = _try_rpc(&pwd);
+ if (rc == PAM_SUCCESS || rc == PAM_PERM_DENIED)
goto cleanup;
- }
-
- info("From %s port %d as %s: unable to determine source job",
- ip_src_str,
- conn.port_src,
- user_name);
- /* Both the single job check and the RPC call have failed to ascertain
- * the correct job to adopt this into. Time for drastic measures */
- rc = _indeterminate_multiple(pamh, steps, pwd.pw_uid, &job_id);
- if (rc == SLURM_SUCCESS) {
- info("From %s port %d as %s: picked job %u",
- ip_src_str,
- conn.port_src,
- user_name,
- job_id);
- rc = _adopt_process(getpid(), job_id);
- } else {
- /* This pam module was worthless, apparently */
- debug("_indeterminate_multiple failed to find a job to adopt this into");
- }
+ /* The source of the connection either didn't reply or couldn't
+ * determine the job ID at the source. Proceed to action_unknown */
+ rc = _action_unknown(pamh, &pwd, steps);
cleanup:
FREE_NULL_LIST(steps);
xfree(buf);
+ xfree(slurm_cgroup_conf);
return rc;
}
diff --git a/doc/html/Makefile.am b/doc/html/Makefile.am
index 4c9488eef84d94325f80bcb1e770844517d5becb..90cc9f757585a50274df807f0ea5b65990659563 100644
--- a/doc/html/Makefile.am
+++ b/doc/html/Makefile.am
@@ -120,7 +120,6 @@ html_DATA = \
ibm_pe_fig2.png \
k_function.gif \
linuxstyles.css \
- lll.gif \
mc_support.gif \
plane_ex1.gif \
plane_ex2.gif \
diff --git a/doc/html/Makefile.in b/doc/html/Makefile.in
index 5c85cd8f3a044d07f88970afbf413be62efa96ae..38c6fcbe8fd35f01b493fbe9dfb05a7fe8b1e954 100644
--- a/doc/html/Makefile.in
+++ b/doc/html/Makefile.in
@@ -550,7 +550,6 @@ html_DATA = \
ibm_pe_fig2.png \
k_function.gif \
linuxstyles.css \
- lll.gif \
mc_support.gif \
plane_ex1.gif \
plane_ex2.gif \
diff --git a/doc/html/burst_buffer.shtml b/doc/html/burst_buffer.shtml
index 5be39365c32ea13eba968749f767a0585d6e37eb..90ffad134f397b6bfb24a5c581e8ca40d68872da 100644
--- a/doc/html/burst_buffer.shtml
+++ b/doc/html/burst_buffer.shtml
@@ -174,7 +174,10 @@ A sample batch script follows:</p>
<p><b>NOTE:</b> The ability to create and destroy persistent burst buffers may be
limited by the "Flags" option in the burst_buffer.conf file.
-By default only privileged users can create or destroy persistent burst buffers.</p>
+See the burst_buffer.conf man page for more information.
+By default only <a href="user_permissions.html">privileged users</a>
+(Slurm operators and administrators)
+can create or destroy persistent burst buffers.</p>
<h2><a name="interactive">Interactive Job Options</a></h2>
@@ -267,6 +270,6 @@ $ scontrol create reservation StartTime=noon duration=60 \
BurstBuffer=cray:20G
</pre>
-<p style="text-align:center;">Last modified 18 September 2015</p>
+<p style="text-align:center;">Last modified 29 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/html/download.shtml b/doc/html/download.shtml
index 249c44351cf27c155203920d91cf87ab8fa78e5c..1292123657e33c0a57c20afa55dd637a83adbc9a 100644
--- a/doc/html/download.shtml
+++ b/doc/html/download.shtml
@@ -29,7 +29,7 @@ a message.</li>
<li><b>MUNGE</b> (recommended)<br>
In order to compile the "auth/munge" authentication plugin for Slurm,
you will need to build and install MUNGE, available from
-<a href="https://code.google.com/p/munge/">https://code.google.com/p/munge/</a> and
+<a href="http://dun.github.io/munge/">http://dun.github.io/munge/</a> and
<a href="http://packages.debian.org/src:munge">Debian</a> and
<a href="http://fedoraproject.org/">Fedora</a> and
<a href="http://packages.ubuntu.com/src:munge">Ubuntu</a>.</li>
@@ -397,6 +397,6 @@ Slurm-based HPC supercomputers. The website of Slurm-web, with screenshots:<br>
</ul>
-<p style="text-align:center;">Last modified 27 July 2015</p>
+<p style="text-align:center;">Last modified 19 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/html/lll.gif b/doc/html/lll.gif
deleted file mode 100644
index d1227248e0ac713e2206273b4009d96d3f5184b3..0000000000000000000000000000000000000000
Binary files a/doc/html/lll.gif and /dev/null differ
diff --git a/doc/html/news.shtml b/doc/html/news.shtml
index c6ae18310009b2c31bfe07f23f92b846ee047745..999402d17cedb185e424caefa198b0c3d4a8c4a1 100644
--- a/doc/html/news.shtml
+++ b/doc/html/news.shtml
@@ -69,21 +69,21 @@ to coordinate activities. Future development plans includes:
<!-- Universitat Jaume I & Universitat Politecnica de Valencia -->
<li>Add support for
<a href="http://slurm.schedmd.com/SUG14/remote_gpu.pdf">Remote CUDA (rCUDA)</a></li>
-<li>Integration with
- <a href="http://en.wikipedia.org/wiki/FlexNet_Publisher">FLEXlm
- (Flexnet Publisher)</a> license management.</li>
<li>Distributed architecture to support the management of resources with Intel
Knight's Landing processors.</li>
-<li>IP communications over InfiniBand network for improved performance.</li>
<li>Fault-tolerance and jobs dynamic adaptation through communication protocol
between Slurm, MPI libraries and the application.</li>
+<li>Improved support for provisioning and virtualization.</li>
+<!-- <li>IP communications over InfiniBand network for improved performance.</li>
+<li>Integration with
+ <a href="http://en.wikipedia.org/wiki/FlexNet_Publisher">FLEXlm
+ (Flexnet Publisher)</a> license management.</li>
<li>Improved support for high-throughput computing (e.g. multiple slurmctld
daemons on a single cluster).</li>
<li>Add Kerberos credential support including credential forwarding
and refresh.</li>
-<li>Improved support for provisioning and virtualization.</li>
-<li>Provide a web-based Slurm administration tool.</li>
+<li>Provide a web-based Slurm administration tool.</li> -->
</ul>
-<p style="text-align:center;">Last modified 31 August 2015</p>
+<p style="text-align:center;">Last modified 22 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/html/quickstart_admin.shtml b/doc/html/quickstart_admin.shtml
index aed44e392a428505415af8b68a668ae6962b8a95..c79de0f9e0881602cce9fef7c3f677a72716ff66 100644
--- a/doc/html/quickstart_admin.shtml
+++ b/doc/html/quickstart_admin.shtml
@@ -38,11 +38,13 @@ those directories will have access rights equal to read/write/execute for
everyone minus the umask value (e.g. umask=0022 generates directories with
permissions of "drwxr-r-x" and mask=0000 generates directories with
permissions of "drwxrwrwx" which is a security problem).</li>
+<li>Type <i>ldconfig -n <library_location></i> so that the Slurm libraries
+can be found by applications that intend to use Slurm APIs directly.</li>
<li>Install the configuration file in <i><sysconfdir>/slurm.conf</i>.<br>
NOTE: You will need to install this configuration file on all nodes of the cluster.</li>
<li>Start the <i>slurmctld</i> and <i>slurmd</i> daemons.</li>
</ol>
-<p>NOTE: Items 3 through 6 can be replaced with</p>
+<p>NOTE: Items 3 through 8 can be replaced with</p>
<ol>
<li><i>rpmbuild -ta slurm*.tar.bz2</i></li>
<li><i>rpm --install <the rpm files></i></li>
@@ -63,6 +65,13 @@ See the README and INSTALL files in the source distribution for more details.
<li>Type <i>make</i> to compile Slurm.</li>
<li>Type <i>make install</i> to install the programs, documentation, libraries,
header files, etc.</li>
+<li>Type <i>ldconfig -n <library_location></i> so that the Slurm libraries
+can be found by applications that intend to use Slurm APIs directly. The
+library location will be a subdirectory of PREFIX (described below) and depend
+upon the system type and configuration, typically lib, lib64 and/or lib32.
+For example, if PREFIX is "/usr" and the subdirectory is "lib64" then you would
+find that a file named "/usr/lib64/libslurm.so" was installed and the command
+"ldconfig -n /usr/lib64" should be executed.</li>
</ol>
<p>A full list of <i>configure</i> options will be returned by the command
<i>configure --help</i>. The most commonly used arguments to the
@@ -840,6 +849,6 @@ options such as mysql and gui tools via a configuration menu.</p>
</pre> <p class="footer"><a href="#top">top</a></p>
-<p style="text-align:center;">Last modified 16 June 2015</p>
+<p style="text-align:center;">Last modified 23 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/html/resource_limits.shtml b/doc/html/resource_limits.shtml
index 447c2559574401d5f4c13fbe21f45c3b92ad74ab..41f6ba59621e56d6572fc4fe4fef59eff5fc22c4 100644
--- a/doc/html/resource_limits.shtml
+++ b/doc/html/resource_limits.shtml
@@ -9,10 +9,10 @@ is strongly recommended before use of this document.</p>
Maui and Moab are not integrated with Slurm's resource limits,
but should use their own resource limits mechanisms.</p>
-<h2>Hierachy</h2>
+<h2>Hierarchy</h2>
<p>Slurm's hierarchical limits are enforced in the following order
- with Job QOS and Partition QOS order being reversable by using the QOS
+ with Job QOS and Partition QOS order being reversible by using the QOS
flag 'OverPartQOS':</p>
<ol>
<li>Partition QOS limit</li>
@@ -24,7 +24,7 @@ but should use their own resource limits mechanisms.</p>
<li>None</li>
</ol>
-<p>Note: If limits are defined at multiple points in this hierachy,
+<p>Note: If limits are defined at multiple points in this hierarchy,
the point in this list where the limit is first defined will be used.
Consider the following example:</p>
<ul>
@@ -171,7 +171,7 @@ specified then no limit will apply.</p>
<li><b>MaxTRESMinsPerJob=</b> A limit of TRES minutes to be used by a job.
If this limit is reached the job will be killed if not running in
- Safe mode, othewise the job will pend until enough time is given to
+ Safe mode, otherwise the job will pend until enough time is given to
complete the job.
</li>
@@ -224,7 +224,7 @@ specified then no limit will apply.</p>
children have to the above system. Can also be the string "parent",
when used on a user this means that the parent association is used
for fairshare. If Fairshare=parent is set on an account, that
- account's children will be effectively reparented for fairshare
+ account's children will be effectively re-parented for fairshare
calculations to the first parent of their parent that is not
Fairshare=parent. Limits remain the same, only it's fairshare value
is affected.
@@ -265,6 +265,6 @@ data maintained in the Slurm database. More information can be found
in the <a href="priority_multifactor.html">priority/multifactor</a>
plugin description.</p>
-<p style="text-align: center;">Last modified 24 September 2015</p>
+<p style="text-align: center;">Last modified 19 October 2015</p>
</ul></body></html>
diff --git a/doc/html/team.shtml b/doc/html/team.shtml
index 725de28e490300aed13ebe870ec8b5cfce3c7173..281f0765b7f5a6a520fca731ea12b7a444906fd8 100644
--- a/doc/html/team.shtml
+++ b/doc/html/team.shtml
@@ -203,6 +203,7 @@ Lead Slurm developers are:
<li>Aleksej Saushev</li>
<li>Uwe Sauter (High Performance Computing Center Stuttgart, Germany)</li>
<li>Chris Scheller (University of Michigan)</li>
+<li>Alejandro (Alex) Sanchez (SchedMD)</li>
<li>Rod Schultz (Bull)</li>
<li>Samuel Senoner (Vienna University of Technology, Austria)</li>
<li>David Singleton</li>
@@ -229,7 +230,7 @@ Lead Slurm developers are:
<li>Daniel M. Weeks (Rensselaer Polytechnic Institute)</li>
<li>Nathan Weeks (Iowa State University)</li>
<li>Andy Wettstein (University of Chicago)</li>
-<li>Tim Wickberg (George Washington University)</li>
+<li>Tim Wickberg (SchedMD)</li>
<li>Chandler Wilkerson (Rice University)</li>
<li>Ramiro Brito Willmersdorf (Universidade Federal de Pemambuco, Brazil)</li>
<li>Jay Windley (Linux NetworX)</li>
@@ -241,6 +242,6 @@ Lead Slurm developers are:
<!-- INDIVIDUALS, PLEASE KEEP IN ALPHABETICAL ORDER -->
</ul>
-<p style="text-align:center;">Last modified 19 October 2015</p>
+<p style="text-align:center;">Last modified 26 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/html/tres.shtml b/doc/html/tres.shtml
index f59b34cc2dc5acd41e13bd18011eb94b3aec0bdf..b4d9820705ef27cb4c774b7a38430355daebab5f 100644
--- a/doc/html/tres.shtml
+++ b/doc/html/tres.shtml
@@ -98,13 +98,13 @@ with:
number of allocated CPUs.
</p>
-<p><b>NOTE:</b> TRESBillingWeights is only used when calcuating fairshare and
+<p><b>NOTE:</b> TRESBillingWeights is only used when calculating fairshare and
doesn't affect job priority directly as it is currently not used for the size of
the job. If you want TRES' to play a role in the job's priority then refer to
the PriorityWeightTRES option.
</p>
-<p><b>NOTE:</b> As with PriorityWeightTRES only TRES definied in
+<p><b>NOTE:</b> As with PriorityWeightTRES only TRES defined in
AccountingStorageTRES are available for TRESBillingWeights.
</p>
</li>
@@ -128,6 +128,6 @@ the PriorityWeightTRES option.
for the requested TRES types. More information about these reports
can be found on the <a href="sreport.html">sreport manpage</a>.
</p>
-<p style="text-align:center;">Last modified 8 September 2015</p>
+<p style="text-align:center;">Last modified 19 October 2015</p>
<!--#include virtual="footer.txt"-->
diff --git a/doc/man/man1/sacct.1 b/doc/man/man1/sacct.1
index bd0d2ed2aaa9f3b6142c5fd322ed2817b92d1612..ab628dc932359f07ce7c3bc0575e8c7b6cefd7d5 100644
--- a/doc/man/man1/sacct.1
+++ b/doc/man/man1/sacct.1
@@ -145,9 +145,8 @@ ReqCPUFreqGov ReqCPUS ReqGRES ReqMem
ReqNodes ReqTRES Reservation ReservationId
Reserved ResvCPU ResvCPURAW Start
State Submit Suspended SystemCPU
-Timelimit TotalCPU TRESAlloc TRESReq
-UID User UserCPU WCKey
-WCKeyID
+Timelimit TotalCPU UID User
+UserCPU WCKey WCKeyID
.ft 1
.fi
diff --git a/doc/man/man1/scontrol.1 b/doc/man/man1/scontrol.1
index 3060f74e7af4a4b576d975aef65718ea8438e7f4..b2a60fe11c4122e4f1f03f23f8daba0610cd753c 100644
--- a/doc/man/man1/scontrol.1
+++ b/doc/man/man1/scontrol.1
@@ -1607,6 +1607,20 @@ exclusive use by the Slurm compute node daemons (slurmd, slurmstepd).
The combined memory limit, in megabytes, on this node for the Slurm compute
node daemons (slurmd, slurmstepd).
+.TP
+The meaning of the memory information is as follows:
+
+.TP
+\fIRealMemory\fP
+The total memory, in MB, on the node.
+
+.TP
+\fIAllocMem\fP
+The total memory, in MB, currently allocated by jobs on the node.
+
+.TP
+\fIFreeMem\fP
+The total memory, in MB, currently free on the node as reported by the OS.
.SH "ENVIRONMENT VARIABLES"
.PP
diff --git a/doc/man/man5/burst_buffer.conf.5 b/doc/man/man5/burst_buffer.conf.5
index 0d620c333363ec0d7e566eca8590dfe4bcbe48c9..5bfa727eae41a2c8de28d104743a3ddc6786f32c 100644
--- a/doc/man/man5/burst_buffer.conf.5
+++ b/doc/man/man5/burst_buffer.conf.5
@@ -66,14 +66,16 @@ Supported options include:
.TP
\fBDisablePersistent\fR
Prevents regular users from being able to create and destroy persistent burst buffers.
-This is the default behaviour, only privileged users can create or destroy persistent burst buffers.
+This is the default behaviour, only privileged users (Slurm operators and
+administrators) can create or destroy persistent burst buffers.
.TP
\fBEmulateCray\fR
Emulating a Cray DataWarp system using the dw_wlm_cli script in the burst_buffer/cray plugin.
.TP
\fBEnablePersistent\fR
Enables regular users to create and destroy persistent burst buffers.
-By default, only privileged users can create or destroy persistent burst buffers.
+By default, only privileged users (Slurm operators and administrators) can
+create or destroy persistent burst buffers.
.RE
.TP
diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index 004fa76d6737b7a66f642de69da896603657e08a..53f4833d947091d38ef40c6020717a9720ab1c0e 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -1113,7 +1113,9 @@ Also see \fBDefaultStorageHost\fR.
The fully qualified file name where job completion records are written
when the \fBJobCompType\fR is "jobcomp/filetxt" or the database where
job completion records are stored when the \fBJobCompType\fR is a
-database.
+database or an url with format http://yourelasticserver:port where job
+completion records are indexed when the \fBJobCompType\fR is
+"jobcomp/elasticsearch".
Also see \fBDefaultStorageLoc\fR.
.TP
@@ -1133,7 +1135,7 @@ Also see \fBDefaultStoragePort\fR.
\fBJobCompType\fR
The job completion logging mechanism type.
Acceptable values at present include "jobcomp/none", "jobcomp/filetxt",
-"jobcomp/mysql", and "jobcomp/script"".
+"jobcomp/mysql", "jobcomp/elasticsearch" and "jobcomp/script"".
The default value is "jobcomp/none", which means that upon job completion
the record of the job is purged from the system. If using the accounting
infrastructure this plugin may not be of interest since the information
@@ -1146,7 +1148,9 @@ parameter.
The value "jobcomp/script" indicates that a script specified by the
\fBJobCompLoc\fR parameter is to be executed with environment variables
indicating the job information.
-
+The value "jobcomp/elasticsearch" indicates that a record of the job
+should be written to an Elasticsearch server specified by the
+\fBJobCompLoc\fR parameter.
.TP
\fBJobCompUser\fR
The user account for accessing the job completion database.
@@ -1999,10 +2003,12 @@ Currently supported options are:
\fBAlloc\fR
If set, the Prolog script will be executed at job allocation. By default,
Prolog is executed just before the task is launched. Therefore, when salloc
-is started, no Prolog is executed. \fBAlloc\fR is useful for preparing things
+is started, no Prolog is executed. Alloc is useful for preparing things
before a user starts to use any allocated resources.
In particular, this flag is needed on a Cray system when cluster compatibility
mode is enabled.
+
+\fBNOTE: Use of the Alloc flag will increase the time required to start jobs.\fR
.TP
\fBContain\fR
At job allocation time, use the ProcTrack plugin to create a job container
@@ -2010,6 +2016,7 @@ on all allocated compute nodes.
This container may be used for user processes not launched under Slurm control,
for example the PAM module may place processes launch through a direct user
login into this container.
+Setting the Contain implicitly sets the Alloc flag.
.TP
\fBNoHold\fR
If set, the Alloc flag should also be set. This will allow for salloc to not
diff --git a/etc/slurmd.service.in b/etc/slurmd.service.in
index 3b1061f6568125ac110a8c2060953d9dddc8c78c..b4703812df979ecd3f20b5f3a4c364d997ab491e 100644
--- a/etc/slurmd.service.in
+++ b/etc/slurmd.service.in
@@ -7,6 +7,7 @@ ConditionPathExists=@sysconfdir@/slurm.conf
Type=forking
ExecStart=@sbindir@/slurmd
PIDFile=/var/run/slurmd.pid
+KillMode=process
[Install]
WantedBy=multi-user.target
diff --git a/slurm.spec b/slurm.spec
index 28ff791f9df20f1b30cd2c74b714213c3a1e5e60..586c47f4095e0461358224743a718fd32a56e7a9 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -726,6 +726,8 @@ test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/launch_slurm.so &&
echo %{_libdir}/slurm/launch_slurm.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/launch_aprun.so &&
echo %{_libdir}/slurm/launch_aprun.so >> $LIST
+test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/mpi_mvapich.so &&
+ echo %{_libdir}/slurm/mpi_mvapich.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/power_cray.so &&
echo %{_libdir}/slurm/power_cray.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/select_bluegene.so &&
@@ -740,6 +742,8 @@ test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/job_container_none.so &&
echo %{_libdir}/slurm/job_container_none.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/task_cray.so &&
echo %{_libdir}/slurm/task_cray.so >> $LIST
+test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/slurmctld_nonstop.so &&
+ echo %{_libdir}/slurm/slurmctld_nonstop.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/switch_cray.so &&
echo %{_libdir}/slurm/switch_cray.so >> $LIST
test -f $RPM_BUILD_ROOT/%{_libdir}/slurm/proctrack_cray.so &&
@@ -946,7 +950,6 @@ rm -rf $RPM_BUILD_ROOT
%{_libdir}/slurm/mpi_mpich1_shmem.so
%{_libdir}/slurm/mpi_mpichgm.so
%{_libdir}/slurm/mpi_mpichmx.so
-%{_libdir}/slurm/mpi_mvapich.so
%{_libdir}/slurm/mpi_openmpi.so
%{_libdir}/slurm/mpi_pmi2.so
%endif
@@ -973,7 +976,6 @@ rm -rf $RPM_BUILD_ROOT
%{_libdir}/slurm/select_cons_res.so
%{_libdir}/slurm/select_linear.so
%{_libdir}/slurm/select_serial.so
-%{_libdir}/slurm/slurmctld_nonstop.so
%{_libdir}/slurm/switch_generic.so
%{_libdir}/slurm/switch_none.so
%{_libdir}/slurm/task_none.so
diff --git a/src/api/job_info.c b/src/api/job_info.c
index 6a64b639bcbaa22cc01e220ddc35428cf21e0a79..97cfc0ffb1f30516298dda1d7e9112733a508bac 100644
--- a/src/api/job_info.c
+++ b/src/api/job_info.c
@@ -693,7 +693,8 @@ line6:
/****** Line 16 ******/
/* Tres should already of been converted at this point from simple */
snprintf(tmp_line, sizeof(tmp_line), "TRES=%s",
- job_ptr->tres_alloc_str);
+ job_ptr->tres_alloc_str ? job_ptr->tres_alloc_str :
+ job_ptr->tres_req_str);
xstrcat(out, tmp_line);
if (one_liner)
xstrcat(out, " ");
diff --git a/src/common/assoc_mgr.c b/src/common/assoc_mgr.c
index e74ec3ec3f9af3589f71c769349d6da32b80c848..2ef983d60d44fe9d1e406279cfae6a7887c6fc75 100644
--- a/src/common/assoc_mgr.c
+++ b/src/common/assoc_mgr.c
@@ -2280,6 +2280,8 @@ extern int assoc_mgr_fill_in_assoc(void *db_conn,
if (!assoc->cluster)
assoc->cluster = ret_assoc->cluster;
+ assoc->def_qos_id = ret_assoc->def_qos_id;
+
if (!assoc->grp_tres_mins)
assoc->grp_tres_mins = ret_assoc->grp_tres_mins;
if (!assoc->grp_tres_run_mins)
diff --git a/src/common/eio.c b/src/common/eio.c
index ba6a7efe7cb206b8695d34eb1a80d8aa143f9507..a57e2bbb8b3726b8e208b4e87edb1a47b4afecf1 100644
--- a/src/common/eio.c
+++ b/src/common/eio.c
@@ -176,12 +176,18 @@ int eio_message_socket_accept(eio_obj_t *obj, List objs)
(socklen_t *)&len)) < 0) {
if (errno == EINTR)
continue;
- if (errno == EAGAIN ||
- errno == ECONNABORTED ||
- errno == EWOULDBLOCK) {
+ if ((errno == EAGAIN) ||
+ (errno == ECONNABORTED) ||
+ (errno == EWOULDBLOCK)) {
return SLURM_SUCCESS;
}
error("Error on msg accept socket: %m");
+ if ((errno == EMFILE) ||
+ (errno == ENFILE) ||
+ (errno == ENOBUFS) ||
+ (errno == ENOMEM)) {
+ return SLURM_SUCCESS;
+ }
obj->shutdown = true;
return SLURM_SUCCESS;
}
diff --git a/src/common/gres.c b/src/common/gres.c
index 37b22025031542bdb2a3c2cf8158aa5732f4303e..6d04fef8683ca9ea06dd590e88611a52dc2b089a 100644
--- a/src/common/gres.c
+++ b/src/common/gres.c
@@ -6623,7 +6623,7 @@ extern char *gres_2_tres_str(List gres_list, bool is_job, bool locked)
continue; /* not tracked */
if (slurmdb_find_tres_count_in_string(
- tres_str, tres_rec->id))
+ tres_str, tres_rec->id) != INFINITE64)
continue; /* already handled */
/* New gres */
diff --git a/src/common/read_config.c b/src/common/read_config.c
index 0f852d67d0683f2fd2bcaf22bf0c8fcec6648fa2..3ff463b9fa5c6d662fa7724ab25c239d93124655 100644
--- a/src/common/read_config.c
+++ b/src/common/read_config.c
@@ -4381,7 +4381,7 @@ extern uint16_t prolog_str2flags(char *prolog_flags)
if (strcasecmp(tok, "Alloc") == 0)
rc |= PROLOG_FLAG_ALLOC;
else if (strcasecmp(tok, "Contain") == 0)
- rc |= PROLOG_FLAG_CONTAIN;
+ rc |= (PROLOG_FLAG_ALLOC | PROLOG_FLAG_CONTAIN);
else if (strcasecmp(tok, "NoHold") == 0)
rc |= PROLOG_FLAG_NOHOLD;
else {
@@ -4443,6 +4443,11 @@ extern char * debug_flags2str(uint64_t debug_flags)
xstrcat(rc, ",");
xstrcat(rc, "BurstBuffer");
}
+ if (debug_flags & DEBUG_FLAG_CPU_FREQ) {
+ if (rc)
+ xstrcat(rc, ",");
+ xstrcat(rc, "CpuFrequency");
+ }
if (debug_flags & DEBUG_FLAG_CPU_BIND) {
if (rc)
xstrcat(rc, ",");
@@ -4563,6 +4568,11 @@ extern char * debug_flags2str(uint64_t debug_flags)
xstrcat(rc, ",");
xstrcat(rc, "NoRealTime");
}
+ if (debug_flags & DEBUG_FLAG_POWER) {
+ if (rc)
+ xstrcat(rc, ",");
+ xstrcat(rc, "Power");
+ }
if (debug_flags & DEBUG_FLAG_PRIO) {
if (rc)
xstrcat(rc, ",");
@@ -4629,16 +4639,7 @@ extern char * debug_flags2str(uint64_t debug_flags)
xstrcat(rc, ",");
xstrcat(rc, "Wiki");
}
- if (debug_flags & DEBUG_FLAG_CPU_FREQ) {
- if (rc)
- xstrcat(rc, ",");
- xstrcat(rc, "CpuFrequency");
- }
- if (debug_flags & DEBUG_FLAG_POWER) {
- if (rc)
- xstrcat(rc, ",");
- xstrcat(rc, "Power");
- }
+
return rc;
}
diff --git a/src/common/slurm_protocol_defs.c b/src/common/slurm_protocol_defs.c
index 73f3fb2777dab35e98c3b8dcf9240243c30ec801..b562c142441f0b15664e2322206b1ac7851d309e 100644
--- a/src/common/slurm_protocol_defs.c
+++ b/src/common/slurm_protocol_defs.c
@@ -675,6 +675,7 @@ extern void slurm_free_job_info_members(job_info_t * job)
xfree(job->array_task_str);
xfree(job->batch_host);
xfree(job->batch_script);
+ xfree(job->burst_buffer);
xfree(job->command);
xfree(job->comment);
xfree(job->dependency);
@@ -701,6 +702,7 @@ extern void slurm_free_job_info_members(job_info_t * job)
xfree(job->std_in);
xfree(job->std_out);
xfree(job->tres_alloc_str);
+ xfree(job->tres_req_str);
xfree(job->wckey);
xfree(job->work_dir);
}
@@ -2880,6 +2882,7 @@ extern void slurm_free_job_step_info_members (job_step_info_t * msg)
{
if (msg != NULL) {
xfree(msg->ckpt_dir);
+ xfree(msg->gres);
xfree(msg->name);
xfree(msg->network);
xfree(msg->nodes);
@@ -2979,6 +2982,7 @@ extern void slurm_free_node_info_members(node_info_t * node)
select_g_select_nodeinfo_free(node->select_nodeinfo);
node->select_nodeinfo = NULL;
xfree(node->version);
+ xfree(node->tres_fmt_str);
/* Do NOT free node, it is an element of an array */
}
}
@@ -3023,12 +3027,14 @@ extern void slurm_free_partition_info_members(partition_info_t * part)
xfree(part->allow_groups);
xfree(part->allow_qos);
xfree(part->alternate);
+ xfree(part->billing_weights_str);
xfree(part->deny_accounts);
xfree(part->deny_qos);
xfree(part->name);
xfree(part->nodes);
xfree(part->node_inx);
xfree(part->qos_char);
+ xfree(part->tres_fmt_str);
}
}
diff --git a/src/common/stepd_api.c b/src/common/stepd_api.c
index fbab6309872da9a86b4c0f270d0f4e7ea3751133..b0f2fe4ebd159803cfa5813e06b5cf36c2c0550e 100644
--- a/src/common/stepd_api.c
+++ b/src/common/stepd_api.c
@@ -93,7 +93,7 @@ _slurm_authorized_user()
* Should be called when a connect() to a socket returns ECONNREFUSED.
* Presumably the ECONNREFUSED means that nothing is attached to the listening
* side of the unix domain socket.
- * If the socket is at least five minutes old, go ahead an unlink it.
+ * If the socket is at least 10 minutes old, then unlink it.
*/
static void
_handle_stray_socket(const char *socket_name)
@@ -120,7 +120,7 @@ _handle_stray_socket(const char *socket_name)
}
now = time(NULL);
- if ((now - buf.st_mtime) > 300) {
+ if ((now - buf.st_mtime) > 600) {
/* remove the socket */
if (unlink(socket_name) == -1) {
if (errno != ENOENT) {
@@ -165,14 +165,15 @@ _step_connect(const char *directory, const char *nodename,
xstrfmtcat(name, "%s/%s_%u.%u", directory, nodename, jobid, stepid);
strcpy(addr.sun_path, name);
- len = strlen(addr.sun_path)+1 + sizeof(addr.sun_family);
+ len = strlen(addr.sun_path) + 1 + sizeof(addr.sun_family);
if (connect(fd, (struct sockaddr *) &addr, len) < 0) {
- error("%s: connect() failed dir %s node %s job %u step %u %m",
+ /* Can indicate race condition at step termination */
+ debug("%s: connect() failed dir %s node %s step %u.%u %m",
__func__, directory, nodename, jobid, stepid);
if (errno == ECONNREFUSED) {
_handle_stray_socket(name);
- if (stepid == NO_VAL)
+ if (stepid == SLURM_BATCH_SCRIPT)
_handle_stray_script(directory, jobid);
}
xfree(name);
@@ -186,7 +187,7 @@ _step_connect(const char *directory, const char *nodename,
static char *
-_guess_nodename()
+_guess_nodename(void)
{
char host[256];
char *nodename = NULL;
diff --git a/src/db_api/job_report_functions.c b/src/db_api/job_report_functions.c
index f1252396892da2e642c78a71ac217405ce95849b..2e0ecdc596f0ab149ccb303d48671e9057d26dab 100644
--- a/src/db_api/job_report_functions.c
+++ b/src/db_api/job_report_functions.c
@@ -215,8 +215,9 @@ static List _process_grouped_report(
if (!job->elapsed)
continue;
- if (!(count = slurmdb_find_tres_count_in_string(
- job->tres_alloc_str, tres_id)))
+ if ((count = slurmdb_find_tres_count_in_string(
+ job->tres_alloc_str, tres_id))
+ == INFINITE64)
continue;
tmp = xstrdup_printf("%"PRIu64, count);
@@ -469,8 +470,9 @@ no_objects:
while ((job_group = list_next(local_itr))) {
uint64_t count;
- if (!(count = slurmdb_find_tres_count_in_string(
- job->tres_alloc_str, tres_id)) ||
+ if (((count = slurmdb_find_tres_count_in_string(
+ job->tres_alloc_str, tres_id))
+ == INFINITE64) ||
(count < job_group->min_size) ||
(count > job_group->max_size))
continue;
diff --git a/src/plugins/accounting_storage/mysql/as_mysql_job.c b/src/plugins/accounting_storage/mysql/as_mysql_job.c
index 3060590cabf762c847e0925ec991f09eeade4844..04a70425319dec6fcf202c2ada1de4eb6e4d5e78 100644
--- a/src/plugins/accounting_storage/mysql/as_mysql_job.c
+++ b/src/plugins/accounting_storage/mysql/as_mysql_job.c
@@ -458,8 +458,6 @@ no_rollup_change:
gres_alloc = slurm_add_slash_to_quotes(job_ptr->gres_alloc);
if (!job_ptr->db_index) {
- if (!begin_time)
- begin_time = submit_time;
query = xstrdup_printf(
"insert into \"%s_%s\" "
"(id_job, mod_time, id_array_job, id_array_task, "
@@ -977,6 +975,12 @@ extern int as_mysql_step_start(mysql_conn_t *mysql_conn,
*/
snprintf(node_list, BUFFER_SIZE, "%s", step_ptr->gres);
nodes = tasks = 1;
+ if (!step_ptr->tres_alloc_str)
+ xstrfmtcat(step_ptr->tres_alloc_str,
+ "%s%u=%u,%u=%u",
+ step_ptr->tres_alloc_str ? "," : "",
+ TRES_CPU, 1,
+ TRES_NODE, 1);
} else {
char *ionodes = NULL, *temp_nodes = NULL;
char temp_bit[BUF_SIZE];
@@ -1001,14 +1005,14 @@ extern int as_mysql_step_start(mysql_conn_t *mysql_conn,
if (step_ptr->cpu_count)
tasks = step_ptr->cpu_count;
else {
- if (!(tasks = slurmdb_find_tres_count_in_string(
- step_ptr->tres_alloc_str,
- TRES_CPU))) {
- if (!(tasks =
- slurmdb_find_tres_count_in_string(
- step_ptr->job_ptr->
- tres_alloc_str,
- TRES_CPU)))
+ if ((tasks = slurmdb_find_tres_count_in_string(
+ step_ptr->tres_alloc_str,
+ TRES_CPU)) == INFINITE64) {
+ if ((tasks =
+ slurmdb_find_tres_count_in_string(
+ step_ptr->job_ptr->
+ tres_alloc_str,
+ TRES_CPU)) == INFINITE64)
tasks = step_ptr->job_ptr->
total_nodes;
}
@@ -1144,14 +1148,14 @@ extern int as_mysql_step_complete(mysql_conn_t *mysql_conn,
if (step_ptr->cpu_count)
tasks = step_ptr->cpu_count;
else {
- if (!(tasks = slurmdb_find_tres_count_in_string(
- step_ptr->tres_alloc_str,
- TRES_CPU))) {
- if (!(tasks =
- slurmdb_find_tres_count_in_string(
- step_ptr->job_ptr->
- tres_alloc_str,
- TRES_CPU)))
+ if ((tasks = slurmdb_find_tres_count_in_string(
+ step_ptr->tres_alloc_str,
+ TRES_CPU)) == INFINITE64) {
+ if ((tasks =
+ slurmdb_find_tres_count_in_string(
+ step_ptr->job_ptr->
+ tres_alloc_str,
+ TRES_CPU)) == INFINITE64)
tasks = step_ptr->job_ptr->
total_nodes;
}
diff --git a/src/plugins/accounting_storage/mysql/as_mysql_jobacct_process.c b/src/plugins/accounting_storage/mysql/as_mysql_jobacct_process.c
index 37736662954196b14947251f2f0a8183b50df67a..cbce5a510f1b2f44b0f2b62f0788b13bfe3ceb8d 100644
--- a/src/plugins/accounting_storage/mysql/as_mysql_jobacct_process.c
+++ b/src/plugins/accounting_storage/mysql/as_mysql_jobacct_process.c
@@ -947,11 +947,13 @@ static int _cluster_get_jobs(mysql_conn_t *mysql_conn,
job->track_steps = 1;
else if (step &&
(xstrcmp(step->stepname, job->jobname) ||
- ((j_cpus = slurmdb_find_tres_count_in_string(
- job->tres_alloc_str, TRES_CPU)) &&
- (s_cpus = slurmdb_find_tres_count_in_string(
- step->tres_alloc_str, TRES_CPU)) &&
- j_cpus != s_cpus)))
+ (((j_cpus = slurmdb_find_tres_count_in_string(
+ job->tres_alloc_str, TRES_CPU))
+ != INFINITE64) &&
+ ((s_cpus = slurmdb_find_tres_count_in_string(
+ step->tres_alloc_str, TRES_CPU))
+ != INFINITE64) &&
+ j_cpus != s_cpus)))
job->track_steps = 1;
}
skip_steps:
@@ -1463,7 +1465,8 @@ extern int setup_job_cond_limits(slurmdb_job_cond_t *job_cond,
job_cond->usage_start);
else
xstrfmtcat(*extra,
- "(t1.time_eligible < %ld "
+ "(t1.time_eligible "
+ "&& t1.time_eligible < %ld "
"&& (t1.time_end >= %ld "
"|| t1.time_end = 0)))",
job_cond->usage_end,
@@ -1474,7 +1477,8 @@ extern int setup_job_cond_limits(slurmdb_job_cond_t *job_cond,
else
xstrcat(*extra, " where (");
xstrfmtcat(*extra,
- "(t1.time_eligible < %ld))",
+ "(t1.time_eligible && "
+ "t1.time_eligible < %ld))",
job_cond->usage_end);
}
}
diff --git a/src/plugins/accounting_storage/mysql/as_mysql_rollup.c b/src/plugins/accounting_storage/mysql/as_mysql_rollup.c
index 5c2befc511de7e46184bc9cbcefcc3758b54194b..faa4324cbdc3d0dd8501816a1b4c0e26c2f0e748 100644
--- a/src/plugins/accounting_storage/mysql/as_mysql_rollup.c
+++ b/src/plugins/accounting_storage/mysql/as_mysql_rollup.c
@@ -307,6 +307,21 @@ static void _add_tres_2_list(List tres_list, char *tres_str, int seconds)
return;
}
+/* This will destroy the *loc_tres given after it is transfered */
+static void _transfer_loc_tres(List *loc_tres, local_id_usage_t *usage)
+{
+ if (!usage)
+ return;
+
+ if (!usage->loc_tres) {
+ usage->loc_tres = *loc_tres;
+ *loc_tres = NULL;
+ } else {
+ _add_job_alloc_time_to_cluster(usage->loc_tres, *loc_tres);
+ FREE_NULL_LIST(*loc_tres);
+ }
+}
+
static void _add_tres_time_2_list(List tres_list, char *tres_str,
int type, int seconds, int suspend_seconds,
bool times_count)
@@ -1109,7 +1124,8 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
"left outer join \"%s_%s\" as step on "
"job.job_db_inx=step.job_db_inx "
"and (step.id_step>=0) "
- "where (job.time_eligible < %ld && "
+ "where (job.time_eligible && "
+ "job.time_eligible < %ld && "
"(job.time_end >= %ld || "
"job.time_end = 0)) "
"group by job.job_db_inx "
@@ -1242,15 +1258,11 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
/* do the cluster allocated calculation */
calc_cluster:
- if (!a_usage)
- loc_tres = list_create(
- _destroy_local_tres_usage);
- else {
- if (!a_usage->loc_tres)
- a_usage->loc_tres = list_create(
- _destroy_local_tres_usage);
- loc_tres = a_usage->loc_tres;
- }
+ /* We need to have this clean for each job
+ * since we add the time to the cluster
+ * individually.
+ */
+ loc_tres = list_create(_destroy_local_tres_usage);
_add_tres_time_2_list(loc_tres, row[JOB_REQ_TRES],
TIME_ALLOC, seconds,
@@ -1299,8 +1311,7 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
/* first figure out the reservation */
if (resv_id) {
if (seconds <= 0) {
- if (!a_usage)
- FREE_NULL_LIST(loc_tres);
+ _transfer_loc_tres(&loc_tres, a_usage);
continue;
}
/* Since we have already added the
@@ -1345,8 +1356,8 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
loc_tres, TIME_ALLOC,
loc_seconds, 1);
}
- if (!a_usage)
- FREE_NULL_LIST(loc_tres);
+
+ _transfer_loc_tres(&loc_tres, a_usage);
continue;
}
@@ -1355,8 +1366,7 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
ever happen.
*/
if (!c_usage) {
- if (!a_usage)
- FREE_NULL_LIST(loc_tres);
+ _transfer_loc_tres(&loc_tres, a_usage);
continue;
}
@@ -1377,9 +1387,10 @@ extern int as_mysql_hourly_rollup(mysql_conn_t *mysql_conn,
loc_tres);
}
- /* The loc_tres isn't needed after this */
- if (!a_usage)
- FREE_NULL_LIST(loc_tres);
+ /* The loc_tres isn't needed after this so
+ * transfer to the association and go on our
+ * merry way. */
+ _transfer_loc_tres(&loc_tres, a_usage);
/* now reserved time */
if (!row_start || (row_start >= c_usage->start)) {
diff --git a/src/plugins/acct_gather_energy/cray/acct_gather_energy_cray.c b/src/plugins/acct_gather_energy/cray/acct_gather_energy_cray.c
index 7ed6983a2621aae1b334815906fba8a5625c7305..ee51e3713ce62f54d2844a2d8578092fc8e07b48 100644
--- a/src/plugins/acct_gather_energy/cray/acct_gather_energy_cray.c
+++ b/src/plugins/acct_gather_energy/cray/acct_gather_energy_cray.c
@@ -270,22 +270,28 @@ extern int acct_gather_energy_p_get_data(enum acct_energy_type data_type,
int rc = SLURM_SUCCESS;
acct_gather_energy_t *energy = (acct_gather_energy_t *)data;
time_t *last_poll = (time_t *)data;
+ uint16_t *sensor_cnt = (uint16_t *)data;
xassert(_run_in_daemon());
switch (data_type) {
case ENERGY_DATA_JOULES_TASK:
+ case ENERGY_DATA_NODE_ENERGY_UP:
if (local_energy->current_watts == NO_VAL)
energy->consumed_energy = NO_VAL;
else
_get_joules_task(energy);
break;
case ENERGY_DATA_STRUCT:
+ case ENERGY_DATA_NODE_ENERGY:
memcpy(energy, local_energy, sizeof(acct_gather_energy_t));
break;
case ENERGY_DATA_LAST_POLL:
*last_poll = local_energy->poll_time;
break;
+ case ENERGY_DATA_SENSOR_CNT:
+ *sensor_cnt = 1;
+ break;
default:
error("acct_gather_energy_p_get_data: unknown enum %d",
data_type);
diff --git a/src/plugins/acct_gather_energy/ibmaem/acct_gather_energy_ibmaem.c b/src/plugins/acct_gather_energy/ibmaem/acct_gather_energy_ibmaem.c
index 572b1a17841810a46de42bf2d132ab3cf82f10f9..043fe08a3a22672fc1a8b79ff679b716a6567686 100644
--- a/src/plugins/acct_gather_energy/ibmaem/acct_gather_energy_ibmaem.c
+++ b/src/plugins/acct_gather_energy/ibmaem/acct_gather_energy_ibmaem.c
@@ -275,6 +275,7 @@ extern int acct_gather_energy_p_get_data(enum acct_energy_type data_type,
switch (data_type) {
case ENERGY_DATA_JOULES_TASK:
+ case ENERGY_DATA_NODE_ENERGY_UP:
if (local_energy->current_watts == NO_VAL)
energy->consumed_energy = NO_VAL;
else
diff --git a/src/plugins/burst_buffer/cray/burst_buffer_cray.c b/src/plugins/burst_buffer/cray/burst_buffer_cray.c
index c6e13eedf45d0d46c58a9d39c5d7c34fff7d93ce..f9cb9dd90a55b1ad444813de699c12f1b290abc5 100644
--- a/src/plugins/burst_buffer/cray/burst_buffer_cray.c
+++ b/src/plugins/burst_buffer/cray/burst_buffer_cray.c
@@ -109,6 +109,19 @@ static bb_state_t bb_state;
static uint32_t last_persistent_id = 1;
static char * state_save_loc = NULL;
+/* These are defined here so when we link with something other than
+ * the slurmctld we will have these symbols defined. They will get
+ * overwritten when linking with the slurmctld.
+ */
+#if defined (__APPLE__)
+int accounting_enforce __attribute__((weak_import)) = 0;
+void *acct_db_conn __attribute__((weak_import)) = NULL;
+#else
+int accounting_enforce = 0;
+void *acct_db_conn = NULL;
+#endif
+
+
/* Description of each Cray DW configuration entry
*/
typedef struct bb_configs {
@@ -200,6 +213,7 @@ static void * _destroy_persistent(void *x);
static void _free_create_args(create_buf_data_t *create_args);
static void _free_script_argv(char **script_argv);
static bb_job_t *_get_bb_job(struct job_record *job_ptr);
+static bool _have_dw_cmd_opts(bb_job_t *bb_job);
static void _job_queue_del(void *x);
static bb_configs_t *_json_parse_configs_array(json_object *jobj, char *key,
int *num);
@@ -231,14 +245,16 @@ static void _purge_bb_files(uint32_t job_id, struct job_record *job_ptr);
static void _purge_vestigial_bufs(void);
static void _python2json(char *buf);
static void _recover_bb_state(void);
+static int _queue_setup(struct job_record *job_ptr, bb_job_t *bb_job);
static int _queue_stage_in(struct job_record *job_ptr, bb_job_t *bb_job);
static int _queue_stage_out(struct job_record *job_ptr);
static void _queue_teardown(uint32_t job_id, uint32_t user_id, bool hurry);
static void _reset_buf_state(uint32_t user_id, uint32_t job_id, char *name,
- int new_state);
+ int new_state, uint64_t buf_size);
static void _save_bb_state(void);
static void _set_assoc_mgr_ptrs(bb_alloc_t *bb_alloc);
static void * _start_pre_run(void *x);
+static void * _start_setup(void *x);
static void * _start_stage_in(void *x);
static void * _start_stage_out(void *x);
static void * _start_teardown(void *x);
@@ -357,7 +373,7 @@ static void _test_config(void)
}
}
-/* Allocate resources to a job and begin stage-in */
+/* Allocate resources to a job and begin setup/stage-in */
static int _alloc_job_bb(struct job_record *job_ptr, bb_job_t *bb_job,
bool job_ready)
{
@@ -384,7 +400,11 @@ static int _alloc_job_bb(struct job_record *job_ptr, bb_job_t *bb_job,
}
}
} else {
- bb_job->state = BB_STATE_STAGED_IN;
+ /* Job uses persistent burst buffer, just run setup */
+ if (bb_job->state < BB_STATE_STAGING_IN) {
+ bb_job->state = BB_STATE_STAGING_IN;
+ rc = _queue_setup(job_ptr, bb_job);
+ }
}
return rc;
@@ -495,6 +515,7 @@ static bb_job_t *_get_bb_job(struct job_record *job_ptr)
bb_job->buf_ptr[inx].size = tmp_cnt;
bb_job->buf_ptr[inx].state = BB_STATE_PENDING;
bb_job->buf_ptr[inx].type = bb_type;
+ bb_job->persist_add += tmp_cnt;
} else if (!strncmp(tok, "destroy_persistent", 17) ||
!strncmp(tok, "delete_persistent", 16)) {
have_bb = true;
@@ -839,6 +860,10 @@ static void _recover_bb_state(void)
bb_alloc = bb_find_name_rec(name, user_id, &bb_state);
}
if (bb_alloc) {
+ if (bb_state.bb_config.debug_flag) {
+ info("Recovered burst buffer %s from user %u",
+ bb_alloc->name, bb_alloc->user_id);
+ }
xfree(bb_alloc->account);
bb_alloc->account = account;
account = NULL;
@@ -1220,6 +1245,145 @@ static int _write_file(char *file_name, char *buf)
return SLURM_SUCCESS;
}
+static int _queue_setup(struct job_record *job_ptr, bb_job_t *bb_job)
+{
+ char *hash_dir = NULL, *job_dir = NULL;
+ char *client_nodes_file_nid = NULL;
+ char **setup_argv;
+ stage_args_t *stage_args;
+ int hash_inx = job_ptr->job_id % 10;
+ pthread_attr_t stage_attr;
+ pthread_t stage_tid = 0;
+ int rc = SLURM_SUCCESS;
+
+ xstrfmtcat(hash_dir, "%s/hash.%d", state_save_loc, hash_inx);
+ (void) mkdir(hash_dir, 0700);
+ xstrfmtcat(job_dir, "%s/job.%u", hash_dir, job_ptr->job_id);
+ if (job_ptr->sched_nodes) {
+ xstrfmtcat(client_nodes_file_nid, "%s/client_nids", job_dir);
+ if (_write_nid_file(client_nodes_file_nid,
+ job_ptr->sched_nodes, job_ptr->job_id))
+ xfree(client_nodes_file_nid);
+ }
+ setup_argv = xmalloc(sizeof(char *) * 20); /* NULL terminated */
+ setup_argv[0] = xstrdup("dw_wlm_cli");
+ setup_argv[1] = xstrdup("--function");
+ setup_argv[2] = xstrdup("setup");
+ setup_argv[3] = xstrdup("--token");
+ xstrfmtcat(setup_argv[4], "%u", job_ptr->job_id);
+ setup_argv[5] = xstrdup("--caller");
+ setup_argv[6] = xstrdup("SLURM");
+ setup_argv[7] = xstrdup("--user");
+ xstrfmtcat(setup_argv[8], "%d", job_ptr->user_id);
+ setup_argv[9] = xstrdup("--capacity");
+ xstrfmtcat(setup_argv[10], "%s:%s",
+ bb_state.bb_config.default_pool,
+ bb_get_size_str(bb_job->total_size));
+ setup_argv[11] = xstrdup("--job");
+ xstrfmtcat(setup_argv[12], "%s/script", job_dir);
+ if (client_nodes_file_nid) {
+#if defined(HAVE_NATIVE_CRAY)
+ setup_argv[13] = xstrdup("--nidlistfile");
+#else
+ setup_argv[13] = xstrdup("--nodehostnamefile");
+#endif
+ setup_argv[14] = xstrdup(client_nodes_file_nid);
+ }
+
+ stage_args = xmalloc(sizeof(stage_args_t));
+ stage_args->job_id = job_ptr->job_id;
+ stage_args->timeout = bb_state.bb_config.stage_in_timeout;
+ stage_args->args1 = setup_argv;
+/* stage_args->args2 = NULL; Nothing to stage-in */
+
+ slurm_attr_init(&stage_attr);
+ if (pthread_attr_setdetachstate(&stage_attr, PTHREAD_CREATE_DETACHED))
+ error("pthread_attr_setdetachstate error %m");
+ while (pthread_create(&stage_tid, &stage_attr, _start_setup,
+ stage_args)) {
+ if (errno != EAGAIN) {
+ error("%s: pthread_create: %m", __func__);
+ _start_setup(stage_args); /* Do in-line */
+ break;
+ }
+ usleep(100000);
+ }
+ slurm_attr_destroy(&stage_attr);
+
+ xfree(hash_dir);
+ xfree(job_dir);
+ xfree(client_nodes_file_nid);
+ return rc;
+}
+
+static void *_start_setup(void *x)
+{
+ stage_args_t *stage_args;
+ char **setup_argv, *resp_msg = NULL, *op = NULL;
+ int rc = SLURM_SUCCESS, status = 0, timeout;
+ slurmctld_lock_t job_write_lock =
+ { NO_LOCK, WRITE_LOCK, NO_LOCK, NO_LOCK };
+ struct job_record *job_ptr;
+ bb_job_t *bb_job;
+ DEF_TIMERS;
+
+ stage_args = (stage_args_t *) x;
+ setup_argv = stage_args->args1;
+
+ if (stage_args->timeout)
+ timeout = stage_args->timeout * 1000;
+ else
+ timeout = DEFAULT_OTHER_TIMEOUT * 1000;
+ op = "setup";
+ START_TIMER;
+ resp_msg = bb_run_script("setup",
+ bb_state.bb_config.get_sys_state,
+ setup_argv, timeout, &status);
+ END_TIMER;
+ if (DELTA_TIMER > 500000) { /* 0.5 secs */
+ info("%s: setup for job %u ran for %s",
+ __func__, stage_args->job_id, TIME_STR);
+ } else if (bb_state.bb_config.debug_flag) {
+ debug("%s: setup for job %u ran for %s",
+ __func__, stage_args->job_id, TIME_STR);
+ }
+ _log_script_argv(setup_argv, resp_msg);
+ if (!WIFEXITED(status) || (WEXITSTATUS(status) != 0)) {
+ error("%s: setup for job %u status:%u response:%s",
+ __func__, stage_args->job_id, status, resp_msg);
+ rc = SLURM_ERROR;
+ }
+ lock_slurmctld(job_write_lock);
+ job_ptr = find_job_record(stage_args->job_id);
+ if (!job_ptr) {
+ error("%s: unable to find job record for job %u",
+ __func__, stage_args->job_id);
+ } else if (rc == SLURM_SUCCESS) {
+ pthread_mutex_lock(&bb_state.bb_mutex);
+ bb_job = bb_job_find(&bb_state, stage_args->job_id);
+ if (bb_job)
+ bb_job->state = BB_STATE_STAGED_IN;
+ pthread_mutex_unlock(&bb_state.bb_mutex);
+ } else {
+ xfree(job_ptr->state_desc);
+ job_ptr->state_reason = FAIL_BURST_BUFFER_OP;
+ xstrfmtcat(job_ptr->state_desc, "%s: %s: %s",
+ plugin_type, op, resp_msg);
+ job_ptr->priority = 0; /* Hold job */
+ pthread_mutex_lock(&bb_state.bb_mutex);
+ bb_job = bb_job_find(&bb_state, stage_args->job_id);
+ if (bb_job)
+ bb_job->state = BB_STATE_COMPLETE;
+ pthread_mutex_unlock(&bb_state.bb_mutex);
+ }
+ unlock_slurmctld(job_write_lock);
+
+ xfree(resp_msg);
+ _free_script_argv(setup_argv);
+ xfree(stage_args);
+ return NULL;
+}
+
static int _queue_stage_in(struct job_record *job_ptr, bb_job_t *bb_job)
{
char *hash_dir = NULL, *job_dir = NULL;
@@ -1818,6 +1982,8 @@ static int _test_size_limit(struct job_record *job_ptr, bb_job_t *bb_job)
xassert(bb_job);
add_space = bb_job->total_size + bb_job->persist_add;
+ if (add_space > bb_state.total_space)
+ return 1;
resv_bb = job_test_bb_resv(job_ptr, now);
if (resv_bb) {
@@ -1832,9 +1998,11 @@ static int _test_size_limit(struct job_record *job_ptr, bb_job_t *bb_job)
resv_space += resv_bb_ptr->used_space;
}
}
+ if ((add_space + resv_space) > bb_state.total_space)
+ return 1;
add_total_space_needed = bb_state.used_space + add_space + resv_space -
- bb_state.total_space;
+ bb_state.total_space;
needed_gres_ptr = xmalloc(sizeof(needed_gres_t) * bb_job->gres_cnt);
for (i = 0; i < bb_job->gres_cnt; i++) {
needed_gres_ptr[i].name = xstrdup(bb_job->gres_ptr[i].name);
@@ -2103,7 +2271,7 @@ static int _parse_bb_opts(struct job_descriptor *job_desc, uint64_t *bb_size,
info("%s: User %d disabled from creating "
"persistent burst buffer",
__func__, submit_uid);
- rc = ESLURM_INVALID_BURST_BUFFER_REQUEST;
+ rc = ESLURM_BURST_BUFFER_PERMISSION;
break;
} else if (!strncmp(tok, "create_persistent", 17)) {
have_bb = true;
@@ -2135,7 +2303,7 @@ static int _parse_bb_opts(struct job_descriptor *job_desc, uint64_t *bb_size,
info("%s: User %d disabled from destroying "
"persistent burst buffer",
__func__, submit_uid);
- rc = ESLURM_INVALID_BURST_BUFFER_REQUEST;
+ rc = ESLURM_BURST_BUFFER_PERMISSION;
break;
} else if (!strncmp(tok, "destroy_persistent", 17)) {
have_bb = true;
@@ -2635,6 +2803,25 @@ fini: xfree(data_buf);
close(path_fd);
}
+/* Return true if #DW options (excludes #BB options) */
+static bool _have_dw_cmd_opts(bb_job_t *bb_job)
+{
+ int i;
+ bb_buf_t *bb_buf;
+
+ xassert(bb_job);
+ if (bb_job->total_size)
+ return true;
+
+ for (i = 0, bb_buf = bb_job->buf_ptr; i < bb_job->buf_cnt;
+ i++, bb_buf++) {
+ if (!bb_buf->create && !bb_buf->destroy)
+ return true;
+ }
+
+ return false;
+}
+
/*
* Secondary validation of a job submit request with respect to burst buffer
* options. Performed after establishing job ID and creating script file.
@@ -2683,6 +2870,11 @@ extern int bb_p_job_validate2(struct job_record *job_ptr, char **err_msg)
return rc;
}
+ if (!_have_dw_cmd_opts(bb_job)) {
+ pthread_mutex_unlock(&bb_state.bb_mutex);
+ return rc;
+ }
+
if (bb_state.bb_config.debug_flag) {
info("%s: %s: %s", plugin_type, __func__,
jobid2fmt(job_ptr, jobid_buf, sizeof(jobid_buf)));
@@ -2975,10 +3167,7 @@ extern int bb_p_job_test_stage_in(struct job_record *job_ptr, bool test_only)
if ((test_only == false) &&
(_test_size_limit(job_ptr, bb_job) == 0) &&
(_alloc_job_bb(job_ptr, bb_job, false) == SLURM_SUCCESS)) {
- if (bb_job->total_size == 0)
- rc = 1; /* Persistent only, space available */
- else
- rc = 0; /* Stage-in job buffer now */
+ rc = 0; /* Setup/stage-in in progress */
}
} else if (bb_job->state == BB_STATE_STAGING_IN) {
rc = 0;
@@ -3353,6 +3542,13 @@ static int _create_bufs(struct job_record *job_ptr, bb_job_t *bb_job,
info("Attempt by job %u to create duplicate "
"persistent burst buffer named %s",
job_ptr->job_id, buf_ptr->name);
+ if (bb_job->persist_add >= bb_alloc->size) {
+ bb_job->persist_add -= bb_alloc->size;
+ } else {
+ error("%s: Persistent buffer size underflow for job %u",
+ __func__, job_ptr->job_id);
+ bb_job->persist_add = 0;
+ }
continue;
}
rc++;
@@ -3487,8 +3683,16 @@ static bool _test_persistent_use_ready(bb_job_t *bb_job,
return true;
}
+/* Reset data structures based upon a change in buffer state
+ * IN user_id - User effected
+ * IN job_id - Job effected
+ * IN name - Buffer name
+ * IN new_state - New buffer state
+ * IN buf_size - Size of created burst buffer only, used to decrement remaining
+ * space requirement for the job
+ */
static void _reset_buf_state(uint32_t user_id, uint32_t job_id, char *name,
- int new_state)
+ int new_state, uint64_t buf_size)
{
bb_buf_t *buf_ptr;
bb_job_t *bb_job;
@@ -3514,6 +3718,17 @@ static void _reset_buf_state(uint32_t user_id, uint32_t job_id, char *name,
if ((old_state == BB_STATE_DELETING) &&
(new_state == BB_STATE_PENDING))
bb_limit_rem(user_id, buf_ptr->size, &bb_state);
+ if ((old_state == BB_STATE_ALLOCATING) &&
+ (new_state == BB_STATE_ALLOCATED) &&
+ ((name[0] < '0') || (name[0] > '9'))) {
+ if (bb_job->persist_add >= buf_size) {
+ bb_job->persist_add -= buf_size;
+ } else {
+ error("%s: Persistent buffer size underflow for job %u",
+ __func__, job_id);
+ bb_job->persist_add = 0;
+ }
+ }
break;
}
@@ -3607,9 +3822,8 @@ static void *_create_persistent(void *x)
resp_msg = NULL;
}
pthread_mutex_lock(&bb_state.bb_mutex);
- _reset_buf_state(create_args->user_id,
- create_args->job_id,
- create_args->name, BB_STATE_PENDING);
+ _reset_buf_state(create_args->user_id, create_args->job_id,
+ create_args->name, BB_STATE_PENDING, 0);
bb_state.last_update_time = time(NULL);
pthread_mutex_unlock(&bb_state.bb_mutex);
unlock_slurmctld(job_write_lock);
@@ -3624,9 +3838,9 @@ static void *_create_persistent(void *x)
__func__, create_args->job_id);
}
pthread_mutex_lock(&bb_state.bb_mutex);
- _reset_buf_state(create_args->user_id,
- create_args->job_id, create_args->name,
- BB_STATE_ALLOCATED);
+ _reset_buf_state(create_args->user_id, create_args->job_id,
+ create_args->name, BB_STATE_ALLOCATED,
+ create_args->size);
bb_alloc = bb_alloc_name_rec(&bb_state, create_args->name,
create_args->user_id);
bb_alloc->size = create_args->size;
@@ -3751,9 +3965,8 @@ static void *_destroy_persistent(void *x)
plugin_type, __func__, resp_msg);
}
pthread_mutex_lock(&bb_state.bb_mutex);
- _reset_buf_state(destroy_args->user_id,
- destroy_args->job_id, destroy_args->name,
- BB_STATE_PENDING);
+ _reset_buf_state(destroy_args->user_id, destroy_args->job_id,
+ destroy_args->name, BB_STATE_PENDING, 0);
bb_state.last_update_time = time(NULL);
pthread_mutex_unlock(&bb_state.bb_mutex);
unlock_slurmctld(job_write_lock);
@@ -3762,9 +3975,8 @@ static void *_destroy_persistent(void *x)
{ READ_LOCK, NO_LOCK, READ_LOCK, NO_LOCK,
NO_LOCK, NO_LOCK, NO_LOCK };
pthread_mutex_lock(&bb_state.bb_mutex);
- _reset_buf_state(destroy_args->user_id,
- destroy_args->job_id, destroy_args->name,
- BB_STATE_DELETED);
+ _reset_buf_state(destroy_args->user_id, destroy_args->job_id,
+ destroy_args->name, BB_STATE_DELETED, 0);
/* Modify internal buffer record for purging */
if (bb_alloc) {
diff --git a/src/plugins/job_submit/lua/job_submit_lua.c b/src/plugins/job_submit/lua/job_submit_lua.c
index 2a08b4e408ae3021b5a83e2b15e92669b2e2d343..e5aaebf5a68592830ba72501ff6f976ed8d47e46 100755
--- a/src/plugins/job_submit/lua/job_submit_lua.c
+++ b/src/plugins/job_submit/lua/job_submit_lua.c
@@ -118,6 +118,18 @@ time_t last_lua_resv_update = (time_t) 0;
static pthread_mutex_t lua_lock = PTHREAD_MUTEX_INITIALIZER;
#endif
+/* These are defined here so when we link with something other than
+ * the slurmctld we will have these symbols defined. They will get
+ * overwritten when linking with the slurmctld.
+ */
+#if defined (__APPLE__)
+int accounting_enforce __attribute__((weak_import)) = 0;
+void *acct_db_conn __attribute__((weak_import)) = NULL;
+#else
+int accounting_enforce = 0;
+void *acct_db_conn = NULL;
+#endif
+
/*****************************************************************************\
* We've provided a simple example of the type of things you can do with this
* plugin. If you develop another plugin that may be of interest to others
@@ -228,8 +240,8 @@ static char *_get_default_account(uint32_t user_id)
memset(&user, 0, sizeof(slurmdb_user_rec_t));
user.uid = user_id;
- if (assoc_mgr_fill_in_user(acct_db_conn,
- &user, 0, NULL) != SLURM_ERROR) {
+ if (assoc_mgr_fill_in_user(acct_db_conn, &user, accounting_enforce,
+ NULL) != SLURM_ERROR) {
return user.default_acct;
} else {
return NULL;
@@ -240,9 +252,8 @@ static char *_get_default_account(uint32_t user_id)
static char *_get_default_qos(uint32_t user_id, char *account, char *partition)
{
slurmdb_assoc_rec_t assoc;
- slurmdb_assoc_rec_t *assoc_ptr;
slurmdb_qos_rec_t qos;
- uint32_t qos_id;
+ uint32_t qos_id = 0;
memset(&assoc, 0, sizeof(slurmdb_assoc_rec_t));
assoc.uid = user_id;
@@ -252,21 +263,18 @@ static char *_get_default_qos(uint32_t user_id, char *account, char *partition)
} else {
assoc.acct = _get_default_account(user_id);
}
- if (assoc_mgr_fill_in_assoc(acct_db_conn, &assoc, 0,
- &assoc_ptr, false) != SLURM_ERROR) {
- qos_id = assoc_ptr->def_qos_id;
- } else {
- return NULL;
- }
- if (!qos_id) {
+ if (assoc_mgr_fill_in_assoc(acct_db_conn, &assoc, accounting_enforce,
+ NULL, false) != SLURM_ERROR)
+ qos_id = assoc.def_qos_id;
+
+ if (!qos_id)
return NULL;
- }
memset(&qos, 0, sizeof(slurmdb_qos_rec_t));
qos.id = qos_id;
- if (assoc_mgr_fill_in_qos(acct_db_conn,
- &qos, 0, NULL, false) != SLURM_ERROR) {
+ if (assoc_mgr_fill_in_qos(acct_db_conn, &qos, accounting_enforce,
+ NULL, false) != SLURM_ERROR) {
return qos.name;
} else {
return NULL;
diff --git a/src/plugins/jobcomp/elasticsearch/jobcomp_elasticsearch.c b/src/plugins/jobcomp/elasticsearch/jobcomp_elasticsearch.c
index f4225f71bf433126aa34c5a8cc041898cc0a7cb3..fd0fe769bddc9c416798d944c25aa5536b667310 100644
--- a/src/plugins/jobcomp/elasticsearch/jobcomp_elasticsearch.c
+++ b/src/plugins/jobcomp/elasticsearch/jobcomp_elasticsearch.c
@@ -110,6 +110,18 @@ const uint32_t plugin_version = SLURM_VERSION_NUMBER;
"\"nodes\":\"%s\",\"total_cpus\":%lu,\"total_nodes\":%lu,"\
"\"derived_exitcode\":%lu,\"exitcode\":%lu,\"state\":\"%s\""
+/* These are defined here so when we link with something other than
+ * the slurmctld we will have these symbols defined. They will get
+ * overwritten when linking with the slurmctld.
+ */
+#if defined (__APPLE__)
+int accounting_enforce __attribute__((weak_import)) = 0;
+void *acct_db_conn __attribute__((weak_import)) = NULL;
+#else
+int accounting_enforce = 0;
+void *acct_db_conn = NULL;
+#endif
+
/* Type for error string table entries */
typedef struct {
int xe_number;
@@ -312,7 +324,7 @@ static int _index_job(const char *jobcomp)
static int error_cnt = 0;
if (log_url == NULL) {
- if (((error_cnt++) % 100) == 0) {
+ if (((++error_cnt) % 100) == 0) {
/* Periodically log errors */
error("%s: Unable to save job state for %d "
"jobs, caching data",
@@ -394,7 +406,7 @@ static int _index_job(const char *jobcomp)
curl_global_cleanup();
if (rc == SLURM_ERROR) {
- if (((error_cnt++) % 100) == 0) {
+ if (((++error_cnt) % 100) == 0) {
/* Periodically log errors */
error("%s: Unable to save job state for %d "
"jobs, caching data",
diff --git a/src/plugins/sched/backfill/backfill.c b/src/plugins/sched/backfill/backfill.c
index 5b7c4828697a578e7c422536fb71d21e83ee36ae..16612a85746444b56f2311d7e7ca541303ce72e6 100644
--- a/src/plugins/sched/backfill/backfill.c
+++ b/src/plugins/sched/backfill/backfill.c
@@ -151,7 +151,7 @@ static void _load_config(void);
static bool _many_pending_rpcs(void);
static bool _more_work(time_t last_backfill_time);
static uint32_t _my_sleep(int usec);
-static int _num_feature_count(struct job_record *job_ptr);
+static int _num_feature_count(struct job_record *job_ptr, bool *has_xor);
static void _reset_job_time_limit(struct job_record *job_ptr, time_t now,
node_space_map_t *node_space);
static int _start_job(struct job_record *job_ptr, bitstr_t *avail_bitmap);
@@ -268,7 +268,7 @@ static bool _many_pending_rpcs(void)
}
/* test if job has feature count specification */
-static int _num_feature_count(struct job_record *job_ptr)
+static int _num_feature_count(struct job_record *job_ptr, bool *has_xor)
{
struct job_details *detail_ptr = job_ptr->details;
int rc = 0;
@@ -282,6 +282,8 @@ static int _num_feature_count(struct job_record *job_ptr)
while ((feat_ptr = (struct feature_record *) list_next(feat_iter))) {
if (feat_ptr->count)
rc++;
+ if (feat_ptr->op_code == FEATURE_OP_XOR)
+ *has_xor = true;
}
list_iterator_destroy(feat_iter);
@@ -298,11 +300,15 @@ static int _try_sched(struct job_record *job_ptr, bitstr_t **avail_bitmap,
uint32_t min_nodes, uint32_t max_nodes,
uint32_t req_nodes, bitstr_t *exc_core_bitmap)
{
- bitstr_t *tmp_bitmap;
+ bitstr_t *low_bitmap = NULL, *tmp_bitmap = NULL;
int rc = SLURM_SUCCESS;
- int feat_cnt = _num_feature_count(job_ptr);
+ bool has_xor = false;
+ int feat_cnt = _num_feature_count(job_ptr, &has_xor);
+ struct job_details *detail_ptr = job_ptr->details;
List preemptee_candidates = NULL;
List preemptee_job_list = NULL;
+ ListIterator feat_iter;
+ struct feature_record *feat_ptr;
if (feat_cnt) {
/* Ideally schedule the job feature by feature,
@@ -312,9 +318,6 @@ static int _try_sched(struct job_record *job_ptr, bitstr_t **avail_bitmap,
* one feature count. It should work fairly well
* in cases where there are multiple feature
* counts. */
- struct job_details *detail_ptr = job_ptr->details;
- ListIterator feat_iter;
- struct feature_record *feat_ptr;
int i = 0, list_size;
uint16_t *feat_cnt_orig = NULL, high_cnt = 0;
@@ -355,6 +358,77 @@ static int _try_sched(struct job_record *job_ptr, bitstr_t **avail_bitmap,
}
list_iterator_destroy(feat_iter);
xfree(feat_cnt_orig);
+ } else if (has_xor) {
+ /* Cache the feature information and test the individual
+ * features, one at a time */
+ struct feature_record feature_base;
+ List feature_cache = detail_ptr->feature_list;
+ time_t low_start = 0;
+
+ detail_ptr->feature_list = list_create(NULL);
+ feature_base.count = 0;
+ feature_base.op_code = FEATURE_OP_END;
+ list_append(detail_ptr->feature_list, &feature_base);
+
+ tmp_bitmap = bit_copy(*avail_bitmap);
+ feat_iter = list_iterator_create(feature_cache);
+ while ((feat_ptr =
+ (struct feature_record *) list_next(feat_iter))) {
+ feature_base.name = feat_ptr->name;
+ if ((job_req_node_filter(job_ptr, *avail_bitmap) ==
+ SLURM_SUCCESS) &&
+ (bit_set_count(*avail_bitmap) >= min_nodes)) {
+ preemptee_candidates =
+ slurm_find_preemptable_jobs(job_ptr);
+ rc = select_g_job_test(job_ptr, *avail_bitmap,
+ min_nodes, max_nodes,
+ req_nodes,
+ SELECT_MODE_WILL_RUN,
+ preemptee_candidates,
+ &preemptee_job_list,
+ exc_core_bitmap);
+ FREE_NULL_LIST(preemptee_job_list);
+ if ((rc == SLURM_SUCCESS) &&
+ ((low_start == 0) ||
+ (low_start > job_ptr->start_time))) {
+ low_start = job_ptr->start_time;
+ low_bitmap = *avail_bitmap;
+ *avail_bitmap = NULL;
+ }
+ }
+ FREE_NULL_BITMAP(*avail_bitmap);
+ *avail_bitmap = bit_copy(tmp_bitmap);
+ }
+ list_iterator_destroy(feat_iter);
+ FREE_NULL_BITMAP(tmp_bitmap);
+ if (low_start) {
+ job_ptr->start_time = low_start;
+ rc = SLURM_SUCCESS;
+ *avail_bitmap = low_bitmap;
+ } else {
+ rc = ESLURM_NODES_BUSY;
+ FREE_NULL_BITMAP(low_bitmap);
+ }
+
+ /* Restore the original feature information */
+ list_destroy(detail_ptr->feature_list);
+ detail_ptr->feature_list = feature_cache;
+ } else if (detail_ptr->feature_list) {
+ if ((job_req_node_filter(job_ptr, *avail_bitmap) !=
+ SLURM_SUCCESS) ||
+ (bit_set_count(*avail_bitmap) < min_nodes)) {
+ rc = ESLURM_NODES_BUSY;
+ } else {
+ preemptee_candidates =
+ slurm_find_preemptable_jobs(job_ptr);
+ rc = select_g_job_test(job_ptr, *avail_bitmap,
+ min_nodes, max_nodes, req_nodes,
+ SELECT_MODE_WILL_RUN,
+ preemptee_candidates,
+ &preemptee_job_list,
+ exc_core_bitmap);
+ FREE_NULL_LIST(preemptee_job_list);
+ }
} else {
/* Try to schedule the job. First on dedicated nodes
* then on shared nodes (if so configured). */
@@ -399,7 +473,6 @@ static int _try_sched(struct job_record *job_ptr, bitstr_t **avail_bitmap,
FREE_NULL_LIST(preemptee_candidates);
return rc;
-
}
/* Terminate backfill_agent */
diff --git a/src/sacct/print.c b/src/sacct/print.c
index 35b187958992541a4d5d901a602cf28552e92510..2abcb50f84d5cc5803b9387f65effa5b9c388688 100644
--- a/src/sacct/print.c
+++ b/src/sacct/print.c
@@ -247,9 +247,9 @@ void print_fields(type_t type, void *object)
if (step->stats.cpu_min != NO_VAL)
got_stats = true;
- if (!(step_cpu_tres_rec_count =
- slurmdb_find_tres_count_in_string(
- step->tres_alloc_str, TRES_CPU)))
+ if ((step_cpu_tres_rec_count =
+ slurmdb_find_tres_count_in_string(
+ step->tres_alloc_str, TRES_CPU)) == INFINITE64)
step_cpu_tres_rec_count =
slurmdb_find_tres_count_in_string(
(job->tres_alloc_str &&
@@ -267,6 +267,12 @@ void print_fields(type_t type, void *object)
break;
}
+ if ((uint64_t)cpu_tres_rec_count == INFINITE64)
+ cpu_tres_rec_count = 0;
+
+ if ((uint64_t)step_cpu_tres_rec_count == INFINITE64)
+ step_cpu_tres_rec_count = 0;
+
list_iterator_reset(print_fields_itr);
while((field = list_next(print_fields_itr))) {
char *tmp_char = NULL, id[FORMAT_STRING_SIZE];
diff --git a/src/scontrol/update_part.c b/src/scontrol/update_part.c
index 79424ad169f5812f8a5422415ef784c57525a270..43d9fbe438aae62d8bc250f56cc7e2799013e64b 100644
--- a/src/scontrol/update_part.c
+++ b/src/scontrol/update_part.c
@@ -448,7 +448,12 @@ scontrol_create_part (int argc, char *argv[])
exit_code = 1;
error("PartitionName must be given.");
return 0;
+ } else if (strcasecmp(part_msg.name, "default") == 0) {
+ exit_code = 1;
+ error("PartitionName cannot be \"DEFAULT\".");
+ return 0;
}
+
if (update_cnt == 0) {
exit_code = 1;
error("No parameters specified");
diff --git a/src/slurmctld/agent.c b/src/slurmctld/agent.c
index 53cbb1da8e9b524d7a77b88a3dd426d431d0c739..6e04f96a2b85c8a699024736e653885276dcc27c 100644
--- a/src/slurmctld/agent.c
+++ b/src/slurmctld/agent.c
@@ -1644,17 +1644,54 @@ static void _set_job_time(struct job_record *job_ptr, uint16_t mail_type,
static void _set_job_term_info(struct job_record *job_ptr, uint16_t mail_type,
char *buf, int buf_len)
{
- uint16_t base_state = job_ptr->job_state & JOB_STATE_BASE;
-
buf[0] = '\0';
+
if ((mail_type == MAIL_JOB_END) || (mail_type == MAIL_JOB_FAIL)) {
- if (WIFEXITED(job_ptr->exit_code)) {
- int exit_code = WEXITSTATUS(job_ptr->exit_code);
- snprintf(buf, buf_len, ", %s, ExitCode %d",
- job_state_string(base_state), exit_code);
+ uint16_t base_state;
+ uint32_t exit_status_min, exit_status_max;
+ int exit_code_min, exit_code_max;
+
+ base_state = job_ptr->job_state & JOB_STATE_BASE;
+ if (job_ptr->array_recs) {
+ exit_status_min = job_ptr->array_recs->min_exit_code;
+ exit_status_max = job_ptr->array_recs->max_exit_code;
+ if (WIFEXITED(exit_status_min) &&
+ WIFEXITED(exit_status_max)) {
+ char *state_string;
+ exit_code_min = WEXITSTATUS(exit_status_min);
+ exit_code_max = WEXITSTATUS(exit_status_max);
+ if ((exit_code_min == 0) && (exit_code_max > 0))
+ state_string = "Mixed";
+ else {
+ state_string =
+ job_state_string(base_state);
+ }
+ snprintf(buf, buf_len, ", %s, ExitCode [%d-%d]",
+ state_string, exit_code_min,
+ exit_code_max);
+ } else if (WIFSIGNALED(exit_status_max)) {
+ exit_code_max = WTERMSIG(exit_status_max);
+ snprintf(buf, buf_len, ", %s, MaxSignal [%d]",
+ "Mixed", exit_code_max);
+ } else if (WIFEXITED(exit_status_max)) {
+ exit_code_max = WEXITSTATUS(exit_status_max);
+ snprintf(buf, buf_len, ", %s, MaxExitCode [%d]",
+ "Mixed", exit_code_max);
+ } else {
+ snprintf(buf, buf_len, ", %s",
+ job_state_string(base_state));
+ }
} else {
- snprintf(buf, buf_len, ", %s",
- job_state_string(base_state));
+ exit_status_max = job_ptr->exit_code;
+ if (WIFEXITED(exit_status_max)) {
+ exit_code_max = WEXITSTATUS(exit_status_max);
+ snprintf(buf, buf_len, ", %s, ExitCode %d",
+ job_state_string(base_state),
+ exit_code_max);
+ } else {
+ snprintf(buf, buf_len, ", %s",
+ job_state_string(base_state));
+ }
}
} else if (buf_len > 0) {
buf[0] = '\0';
@@ -1676,9 +1713,24 @@ extern void mail_job_info (struct job_record *job_ptr, uint16_t mail_type)
else
mi->user_name = xstrdup(job_ptr->mail_user);
+ /* Use job array master record, if available */
+ if ((job_ptr->array_task_id != NO_VAL) && !job_ptr->array_recs) {
+ struct job_record *master_job_ptr;
+ master_job_ptr = find_job_record(job_ptr->array_job_id);
+ if (master_job_ptr && master_job_ptr->array_recs)
+ job_ptr = master_job_ptr;
+ }
+
_set_job_time(job_ptr, mail_type, job_time, sizeof(job_time));
_set_job_term_info(job_ptr, mail_type, term_msg, sizeof(term_msg));
- if (job_ptr->array_task_id != NO_VAL) {
+ if (job_ptr->array_recs) {
+ mi->message = xstrdup_printf("SLURM Job_id=%u_* (%u) Name=%s "
+ "%s%s%s",
+ job_ptr->array_job_id,
+ job_ptr->job_id, job_ptr->name,
+ _mail_type_str(mail_type),
+ job_time, term_msg);
+ } else if (job_ptr->array_task_id != NO_VAL) {
mi->message = xstrdup_printf("SLURM Job_id=%u_%u (%u) Name=%s "
"%s%s%s",
job_ptr->array_job_id,
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index 071770c26af6dc6ee45193dc6802caa2398de0cf..78fc1e913c5a0d6c5185fae8cc31830b5501595a 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -175,6 +175,7 @@ static void _dump_job_state(struct job_record *dump_job_ptr, Buf buffer);
static int _find_batch_dir(void *x, void *key);
static void _get_batch_job_dir_ids(List batch_dirs);
static time_t _get_last_state_write_time(void);
+static void _job_array_comp(struct job_record *job_ptr, bool was_running);
static int _job_create(job_desc_msg_t * job_specs, int allocate, int will_run,
struct job_record **job_rec_ptr, uid_t submit_uid,
char **err_msg, uint16_t protocol_version);
@@ -4617,8 +4618,10 @@ extern int job_str_signal(char *job_id_str, uint16_t signal, uint16_t flags,
/* Master job record, even wihtout tasks,
* counts as one job record */
job_count -= (orig_task_cnt - 1);
- } else
+ } else {
+ _job_array_comp(job_ptr, false);
job_count -= (orig_task_cnt - new_task_count);
+ }
/* Set the task_cnt here since
* job_completion_logger needs the total
@@ -7273,7 +7276,6 @@ extern void job_set_req_tres(
if (!assoc_mgr_locked)
assoc_mgr_lock(&locks);
- xfree(job_ptr->tres_req_cnt);
job_ptr->tres_req_cnt = xmalloc(sizeof(uint64_t) * g_tres_count);
if (job_ptr->details) {
@@ -7335,6 +7337,7 @@ extern void job_set_alloc_tres(struct job_record *job_ptr,
xfree(job_ptr->tres_alloc_str);
xfree(job_ptr->tres_alloc_cnt);
+ xfree(job_ptr->tres_fmt_alloc_str);
/* We only need to do this on non-pending jobs */
if (IS_JOB_PENDING(job_ptr))
@@ -7342,7 +7345,6 @@ extern void job_set_alloc_tres(struct job_record *job_ptr,
if (!assoc_mgr_locked)
assoc_mgr_lock(&locks);
- xfree(job_ptr->tres_alloc_cnt);
job_ptr->tres_alloc_cnt = xmalloc(
sizeof(uint64_t) * slurmctld_tres_cnt);
@@ -7381,11 +7383,9 @@ extern void job_set_alloc_tres(struct job_record *job_ptr,
true);
/* now that the array is filled lets make the string from it */
- xfree(job_ptr->tres_alloc_str);
job_ptr->tres_alloc_str = assoc_mgr_make_tres_str_from_array(
job_ptr->tres_alloc_cnt, TRES_STR_FLAG_SIMPLE, true);
- xfree(job_ptr->tres_fmt_alloc_str);
job_ptr->tres_fmt_alloc_str = assoc_mgr_make_tres_str_from_array(
job_ptr->tres_alloc_cnt, 0, true);
@@ -10602,11 +10602,11 @@ static int _update_job(struct job_record *job_ptr, job_desc_msg_t * job_specs,
job_ptr->state_reason = WAIT_HELD;
xfree(job_ptr->state_desc);
}
- } else if (job_specs->priority == INFINITE
- && job_ptr->state_reason != WAIT_HELD_USER) {
- /* If the job was already released ignore another
- * release request.
- */
+ } else if ((job_ptr->priority != 0) &&
+ (job_specs->priority == INFINITE) &&
+ (job_ptr->state_reason != WAIT_HELD_USER)) {
+ /* If the job was already released, ignore another
+ * release request. */
debug("%s: job %d already release ignoring request",
__func__, job_ptr->job_id);
} else {
@@ -11958,7 +11958,7 @@ static void _purge_missing_jobs(int node_inx, time_t now)
node_boot_time = node_ptr->boot_time - (msg_timeout + 5);
}
batch_startup_time = now - batch_start_timeout;
- batch_startup_time -= msg_timeout;
+ batch_startup_time -= MIN(DEFAULT_MSG_TIMEOUT, msg_timeout);
job_iterator = list_iterator_create(job_list);
while ((job_ptr = (struct job_record *) list_next(job_iterator))) {
@@ -12645,7 +12645,7 @@ extern bool job_array_start_test(struct job_record *job_ptr)
return true;
}
-static void _job_array_comp(struct job_record *job_ptr)
+static void _job_array_comp(struct job_record *job_ptr, bool was_running)
{
struct job_record *base_job_ptr;
uint32_t status;
@@ -12672,7 +12672,8 @@ static void _job_array_comp(struct job_record *job_ptr)
MAX(status, base_job_ptr->
array_recs->max_exit_code);
}
- if (base_job_ptr->array_recs->tot_run_tasks)
+ if (was_running &&
+ base_job_ptr->array_recs->tot_run_tasks)
base_job_ptr->array_recs->tot_run_tasks--;
base_job_ptr->array_recs->tot_comp_tasks++;
}
@@ -12695,6 +12696,8 @@ extern void job_completion_logger(struct job_record *job_ptr, bool requeue)
(void) bb_g_job_cancel(job_ptr);
}
+ _job_array_comp(job_ptr, true);
+
if (!IS_JOB_RESIZING(job_ptr) &&
((job_ptr->array_task_id == NO_VAL) ||
test_job_array_finished(job_ptr->array_job_id))) {
@@ -12721,8 +12724,6 @@ extern void job_completion_logger(struct job_record *job_ptr, bool requeue)
}
}
- _job_array_comp(job_ptr);
-
g_slurm_jobcomp_write(job_ptr);
/* When starting the resized job everything is taken care of
diff --git a/src/slurmctld/licenses.c b/src/slurmctld/licenses.c
index 85818bcf0b264ba69a8a41203c65dba51456bac6..abb95b2caec44251cc14ccfd658c2da3c37fe423 100644
--- a/src/slurmctld/licenses.c
+++ b/src/slurmctld/licenses.c
@@ -896,9 +896,8 @@ extern char *licenses_2_tres_str(List license_list)
continue; /* not tracked */
if (slurmdb_find_tres_count_in_string(
- tres_str, tres_rec->id))
+ tres_str, tres_rec->id) != INFINITE64)
continue; /* already handled */
-
/* New license */
xstrfmtcat(tres_str, "%s%u=%"PRIu64,
tres_str ? "," : "",
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 8564a6d2e40899f3db44acf4d6ae5884ca8671fe..2c82dd1d5873795fe440934e11e8390cde865d86 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -1002,7 +1002,7 @@ _get_req_features(struct node_set *node_set_ptr, int node_set_size,
#if 0
{
char *tmp_str = bitmap2node_name(job_ptr->details->req_node_bitmap);
- info("job %u requires %d:%d:%d nodes %s err:%u",
+ info("job %u requires %d:%d:%d req_nodes:%s err:%u",
job_ptr->job_id, min_nodes, req_nodes, max_nodes,
tmp_str, error_code);
xfree(tmp_str);
@@ -1492,13 +1492,15 @@ _pick_best_nodes(struct node_set *node_set_ptr, int node_set_size,
tried_sched = false; /* need to test these nodes */
if ((switch_record_cnt > 1) &&
- ((i+1) < node_set_size)) {
+ ((i+1) < node_set_size) &&
+ (min_feature == max_feature)) {
/* Keep accumulating to optimize topology */
continue;
}
- if ((shared || preempt_flag) &&
- ((i+1) < node_set_size) &&
+ if ((shared || preempt_flag) &&
+ ((i+1) < node_set_size) &&
+ (min_feature == max_feature) &&
(node_set_ptr[i].weight ==
node_set_ptr[i+1].weight)) {
/* Keep accumulating so we can pick the
diff --git a/src/slurmctld/proc_req.c b/src/slurmctld/proc_req.c
index 0353756fbfcaa6d65cfd9fa56eeb0372fca0bc3b..914c87954c0edc4485dff366253490698dae7550 100644
--- a/src/slurmctld/proc_req.c
+++ b/src/slurmctld/proc_req.c
@@ -2123,9 +2123,12 @@ static void _slurm_rpc_complete_batch_script(slurm_msg_t * msg, bool locked)
batch_step.name = "batch";
batch_step.select_jobinfo = job_ptr->select_jobinfo;
+ step_set_alloc_tres(&batch_step, 1, false, false);
+
jobacct_storage_g_step_start(acct_db_conn, &batch_step);
jobacct_storage_g_step_complete(acct_db_conn, &batch_step);
FREE_NULL_BITMAP(batch_step.step_node_bitmap);
+ xfree(batch_step.tres_alloc_str);
}
#ifdef HAVE_FRONT_END
diff --git a/src/slurmctld/reservation.c b/src/slurmctld/reservation.c
index 06c8d45351dda8bf5779dbf3bf54ce7c50074a19..abb2e27b93b2e5c0b5debe2bdd8b7564430cdf2c 100644
--- a/src/slurmctld/reservation.c
+++ b/src/slurmctld/reservation.c
@@ -3742,7 +3742,8 @@ static int _select_nodes(resv_desc_msg_t *resv_desc_ptr,
FREE_NULL_BITMAP(feature_bitmap);
}
- if ((resv_desc_ptr->flags & RESERVE_FLAG_MAINT) == 0) {
+ if (((resv_desc_ptr->flags & RESERVE_FLAG_MAINT) == 0) &&
+ ((resv_desc_ptr->flags & RESERVE_FLAG_SPEC_NODES) == 0)) {
/* Nodes must be available */
bit_and(node_bitmap, avail_node_bitmap);
}
diff --git a/src/slurmctld/slurmctld.h b/src/slurmctld/slurmctld.h
index 81a7830ce6d3d3f50b7f796734e854b0d5e51756..7b175d33c7fd53652b603ed3228360ad0ca2396b 100644
--- a/src/slurmctld/slurmctld.h
+++ b/src/slurmctld/slurmctld.h
@@ -2124,7 +2124,7 @@ extern void start_power_mgr(pthread_t *thread_id);
* IN node_name - name of node which has completed epilog
*/
extern int step_epilog_complete(struct job_record *job_ptr,
- char *node_name);
+ char *node_name);
/*
* step_partial_comp - Note the completion of a job step on at least
@@ -2138,6 +2138,14 @@ extern int step_epilog_complete(struct job_record *job_ptr,
extern int step_partial_comp(step_complete_msg_t *req, uid_t uid,
int *rem, uint32_t *max_rc);
+/*
+ * step_set_alloc_tres - set the tres up when allocating the step.
+ * Only set when job is running.
+ * NOTE: job write lock must be locked before calling this */
+extern void step_set_alloc_tres(
+ struct step_record *step_ptr, uint32_t node_count,
+ bool assoc_mgr_locked, bool make_formatted);
+
/* Update time stamps for job step suspend */
extern void suspend_job_step(struct job_record *job_ptr);
diff --git a/src/slurmctld/step_mgr.c b/src/slurmctld/step_mgr.c
index 7570abce1e7e51b7baabf3b0eb48f29d9c4762b9..45c5377ee1ccaa7922ad54dbb686f74a77ccd6bf 100644
--- a/src/slurmctld/step_mgr.c
+++ b/src/slurmctld/step_mgr.c
@@ -2114,16 +2114,12 @@ step_create(job_step_create_request_msg_t *step_specs,
bitstr_t *nodeset;
int cpus_per_task, ret_code, i;
uint32_t node_count = 0;
- uint64_t cpu_count, tres_count;
time_t now = time(NULL);
char *step_node_list = NULL;
uint32_t orig_cpu_count;
List step_gres_list = (List) NULL;
dynamic_plugin_data_t *select_jobinfo = NULL;
uint32_t task_dist;
- char *tmp_tres_str = NULL;
- assoc_mgr_lock_t locks = { READ_LOCK, NO_LOCK, NO_LOCK, NO_LOCK,
- READ_LOCK, NO_LOCK, NO_LOCK };
#ifdef HAVE_ALPS_CRAY
uint32_t resv_id = 0;
@@ -2534,46 +2530,8 @@ step_create(job_step_create_request_msg_t *step_specs,
select_g_step_start(step_ptr);
-#ifdef HAVE_BG_L_P
- /* Only L and P use this code */
- if (step_ptr->job_ptr->details)
- cpu_count = (uint64_t)step_ptr->job_ptr->details->min_cpus;
- else
- cpu_count = (uint64_t)step_ptr->job_ptr->cpu_cnt;
-#else
- if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt)
- cpu_count = (uint64_t)step_ptr->job_ptr->total_cpus;
- else
- cpu_count = (uint64_t)step_ptr->cpu_count;
-#endif
- xfree(step_ptr->tres_alloc_str);
-
- tres_count = (uint64_t)step_ptr->pn_min_memory;
- if (tres_count & MEM_PER_CPU) {
- tres_count &= (~MEM_PER_CPU);
- tres_count *= cpu_count;
- } else
- tres_count *= node_count;
- xstrfmtcat(step_ptr->tres_alloc_str,
- "%s%u=%"PRIu64",%u=%"PRIu64",%u=%u",
- step_ptr->tres_alloc_str ? "," : "",
- TRES_CPU, cpu_count,
- TRES_MEM, tres_count,
- TRES_NODE, node_count);
-
- if ((tmp_tres_str = gres_2_tres_str(step_ptr->gres_list, 0, true))) {
- xstrfmtcat(step_ptr->tres_alloc_str, "%s%s",
- step_ptr->tres_alloc_str ? "," : "",
- tmp_tres_str);
- xfree(tmp_tres_str);
- }
-
- xfree(step_ptr->tres_fmt_alloc_str);
- assoc_mgr_lock(&locks);
- step_ptr->tres_fmt_alloc_str = slurmdb_make_tres_string_from_simple(
- step_ptr->tres_alloc_str, assoc_mgr_tres_list);
- assoc_mgr_unlock(&locks);
+ step_set_alloc_tres(step_ptr, node_count, false, true);
jobacct_storage_g_step_start(acct_db_conn, step_ptr);
return SLURM_SUCCESS;
@@ -3402,6 +3360,83 @@ extern int step_partial_comp(step_complete_msg_t *req, uid_t uid,
return SLURM_SUCCESS;
}
+/*
+ * step_set_alloc_tres - set the tres up when allocating the step.
+ * Only set when job is running.
+ * NOTE: job write lock must be locked before calling this */
+extern void step_set_alloc_tres(
+ struct step_record *step_ptr, uint32_t node_count,
+ bool assoc_mgr_locked, bool make_formatted)
+{
+ uint64_t cpu_count = 1, mem_count = 1;
+ char *tmp_tres_str = NULL;
+
+ xassert(step_ptr);
+
+ xfree(step_ptr->tres_alloc_str);
+ xfree(step_ptr->tres_fmt_alloc_str);
+
+ if ((step_ptr->step_id == SLURM_BATCH_SCRIPT) &&
+ step_ptr->job_ptr->job_resrcs) {
+ /* get the cpus and memory on the first node */
+ if (step_ptr->job_ptr->job_resrcs->cpus)
+ cpu_count = step_ptr->job_ptr->job_resrcs->cpus[0];
+ if (step_ptr->job_ptr->job_resrcs->memory_allocated)
+ mem_count = step_ptr->job_ptr->job_resrcs->
+ memory_allocated[0];
+ } else {
+#ifdef HAVE_BG_L_P
+ /* Only L and P use this code */
+ if (step_ptr->job_ptr->details)
+ cpu_count =
+ (uint64_t)step_ptr->job_ptr->details->min_cpus;
+ else
+ cpu_count = (uint64_t)step_ptr->job_ptr->cpu_cnt;
+#else
+ if (!step_ptr->step_layout || !step_ptr->step_layout->task_cnt)
+ cpu_count = (uint64_t)step_ptr->job_ptr->total_cpus;
+ else
+ cpu_count = (uint64_t)step_ptr->cpu_count;
+#endif
+ mem_count = (uint64_t)step_ptr->pn_min_memory;
+ if (mem_count & MEM_PER_CPU) {
+ mem_count &= (~MEM_PER_CPU);
+ mem_count *= cpu_count;
+ } else
+ mem_count *= node_count;
+ }
+
+ xstrfmtcat(step_ptr->tres_alloc_str,
+ "%s%u=%"PRIu64",%u=%"PRIu64",%u=%u",
+ step_ptr->tres_alloc_str ? "," : "",
+ TRES_CPU, cpu_count,
+ TRES_MEM, mem_count,
+ TRES_NODE, node_count);
+
+ if ((tmp_tres_str = gres_2_tres_str(step_ptr->gres_list, 0, true))) {
+ xstrfmtcat(step_ptr->tres_alloc_str, "%s%s",
+ step_ptr->tres_alloc_str ? "," : "",
+ tmp_tres_str);
+ xfree(tmp_tres_str);
+ }
+
+ if (make_formatted) {
+ assoc_mgr_lock_t locks = { NO_LOCK, NO_LOCK, NO_LOCK, NO_LOCK,
+ READ_LOCK, NO_LOCK, NO_LOCK };
+ if (!assoc_mgr_locked)
+ assoc_mgr_lock(&locks);
+
+ step_ptr->tres_fmt_alloc_str =
+ slurmdb_make_tres_string_from_simple(
+ step_ptr->tres_alloc_str, assoc_mgr_tres_list);
+
+ if (!assoc_mgr_locked)
+ assoc_mgr_unlock(&locks);
+ }
+
+ return;
+}
+
/* convert a range of nodes allocated to a step to a hostlist with
* names of those nodes */
static hostlist_t _step_range_to_hostlist(struct step_record *step_ptr,
diff --git a/src/slurmd/common/proctrack.c b/src/slurmd/common/proctrack.c
index 68b88c7ca0abf7f0a217c241f9a68507aa28dad9..995c6533f117a5c3f2b0b19c8b2649e6520feeac 100644
--- a/src/slurmd/common/proctrack.c
+++ b/src/slurmd/common/proctrack.c
@@ -223,6 +223,13 @@ static bool _test_core_dumping(char* stat_fname)
/* split into "PID (cmd" and "<rest>" */
str_ptr = (char *)strrchr(proc_stat, ')');
+ if (str_ptr == NULL) {
+ error("\
+%s: unexpected format of %s (%s) bracket missing?", __func__,
+ stat_fname, proc_stat);
+ xfree(proc_stat);
+ return false;
+ }
*str_ptr = '\0'; /* replace trailing ')' with NULL */
/* parse these two strings separately, skipping the leading "(". */
memset (cmd, 0, sizeof(cmd));
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index 0cbfdef9bb2564ae6c7292bd0725e45197f4509e..7f5988ccac427e4b0af504e875ea6cce6b199fa4 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -800,10 +800,12 @@ _forkexec_slurmstepd(uint16_t type, void *req,
#ifndef SLURMSTEPD_MEMCHECK
i = read(to_slurmd[0], &rc, sizeof(int));
if (i < 0) {
- error("Can not read return code from slurmstepd: %m");
+ error("\
+%s: Can not read return code from slurmstepd got %d: %m", __func__, i);
rc = SLURM_FAILURE;
} else if (i != sizeof(int)) {
- error("slurmstepd failed to send return code");
+ error("\
+%s: slurmstepd failed to send return code got %d: %m", __func__, i);
rc = SLURM_FAILURE;
} else {
int delta_time = time(NULL) - start_time;
@@ -1724,16 +1726,15 @@ static void _rpc_prolog(slurm_msg_t *msg)
rc = ESLURMD_PROLOG_FAILED;
}
- if (slurmctld_conf.prolog_flags & PROLOG_FLAG_CONTAIN)
- _make_prolog_mem_container(msg);
-
- if (container_g_create(req->job_id))
- error("container_g_create(%u): %m", req->job_id);
-
slurm_mutex_lock(&prolog_mutex);
first_job_run = !slurm_cred_jobid_cached(conf->vctx, req->job_id);
-
if (first_job_run) {
+ if (slurmctld_conf.prolog_flags & PROLOG_FLAG_CONTAIN)
+ _make_prolog_mem_container(msg);
+
+ if (container_g_create(req->job_id))
+ error("container_g_create(%u): %m", req->job_id);
+
slurm_cred_insert_jobid(conf->vctx, req->job_id);
_add_job_running_prolog(req->job_id);
slurm_mutex_unlock(&prolog_mutex);
@@ -2518,7 +2519,7 @@ _rpc_health_check(slurm_msg_t *msg)
* slurmctld in hopes of avoiding having the node set DOWN due to
* slurmd paging and not being able to respond in a timely fashion. */
if (slurm_send_rc_msg(msg, rc) < 0) {
- error("Error responding to ping: %m");
+ error("Error responding to health check: %m");
send_registration_msg(SLURM_SUCCESS, false);
}
@@ -2563,7 +2564,7 @@ _rpc_acct_gather_update(slurm_msg_t *msg)
* due to slurmd paging and not being able to respond in a
* timely fashion. */
if (slurm_send_rc_msg(msg, rc) < 0) {
- error("Error responding to ping: %m");
+ error("Error responding to account gather: %m");
send_registration_msg(SLURM_SUCCESS, false);
}
} else {
@@ -2669,7 +2670,7 @@ _signal_jobstep(uint32_t jobid, uint32_t stepid, uid_t req_uid,
fd = stepd_connect(conf->spooldir, conf->node_name, jobid, stepid,
&protocol_version);
if (fd == -1) {
- debug("signal for nonexistant %u.%u stepd_connect failed: %m",
+ debug("signal for nonexistent %u.%u stepd_connect failed: %m",
jobid, stepid);
return ESLURM_INVALID_JOB_ID;
}
@@ -2756,7 +2757,7 @@ _rpc_checkpoint_tasks(slurm_msg_t *msg)
fd = stepd_connect(conf->spooldir, conf->node_name,
req->job_id, req->job_step_id, &protocol_version);
if (fd == -1) {
- debug("checkpoint for nonexistant %u.%u stepd_connect "
+ debug("checkpoint for nonexistent %u.%u stepd_connect "
"failed: %m", req->job_id, req->job_step_id);
rc = ESLURM_INVALID_JOB_ID;
goto done;
@@ -2802,7 +2803,7 @@ _rpc_terminate_tasks(slurm_msg_t *msg)
fd = stepd_connect(conf->spooldir, conf->node_name,
req->job_id, req->job_step_id, &protocol_version);
if (fd == -1) {
- debug("kill for nonexistant job %u.%u stepd_connect "
+ debug("kill for nonexistent job %u.%u stepd_connect "
"failed: %m", req->job_id, req->job_step_id);
rc = ESLURM_INVALID_JOB_ID;
goto done;
diff --git a/src/slurmd/slurmstepd/req.c b/src/slurmd/slurmstepd/req.c
index 92d7c95c64bd66b8680efaf55090876c3b8251eb..7a6ebb623d30b473fb8fa8b7b41cf2f84553c0f4 100644
--- a/src/slurmd/slurmstepd/req.c
+++ b/src/slurmd/slurmstepd/req.c
@@ -153,7 +153,7 @@ _create_socket(const char *name)
if (bind(fd, (struct sockaddr *) &addr, len) < 0)
return -2;
- if (listen(fd, 5) < 0)
+ if (listen(fd, 32) < 0)
return -3;
return fd;
@@ -274,7 +274,7 @@ msg_thr_create(stepd_step_rec_t *job)
* This gives connection threads a chance to complete any pending
* RPCs before the slurmstepd exits.
*/
-static void _wait_for_connections()
+static void _wait_for_connections(void)
{
struct timespec ts = {0, 0};
int rc = 0;
@@ -326,12 +326,18 @@ _msg_socket_accept(eio_obj_t *obj, List objs)
(socklen_t *)&len)) < 0) {
if (errno == EINTR)
continue;
- if (errno == EAGAIN
- || errno == ECONNABORTED
- || errno == EWOULDBLOCK) {
+ if ((errno == EAGAIN) ||
+ (errno == ECONNABORTED) ||
+ (errno == EWOULDBLOCK)) {
return SLURM_SUCCESS;
}
error("Error on msg accept socket: %m");
+ if ((errno == EMFILE) ||
+ (errno == ENFILE) ||
+ (errno == ENOBUFS) ||
+ (errno == ENOMEM)) {
+ return SLURM_SUCCESS;
+ }
obj->shutdown = true;
return SLURM_SUCCESS;
}
diff --git a/src/smap/smap.c b/src/smap/smap.c
index 1501acb96df077add0dca4f31eeb1816449f9a60..021b38df7abab6a62f9449113729b8c4a3a45307 100644
--- a/src/smap/smap.c
+++ b/src/smap/smap.c
@@ -184,13 +184,11 @@ int main(int argc, char *argv[])
if (params.cluster_dims == 4) {
startx = width;
- COLS -= 2;
- width = COLS - width;
+ width = COLS - width - 2;
height = LINES;
} else if (params.cluster_dims == 3) {
startx = width;
- COLS -= 2;
- width = COLS - width;
+ width = COLS - width - 2;
height = LINES;
} else {
startx = 0;
@@ -476,11 +474,8 @@ static void *_resize_handler(int sig)
delwin(text_win);
endwin();
- COLS = 0;
- LINES = 0;
initscr();
doupdate(); /* update now to make sure we get the new size */
- getmaxyx(stdscr, LINES, COLS);
if (params.cluster_dims == 4) {
height = dim_size[2] * dim_size[3] + dim_size[2] + 3;
@@ -508,13 +503,11 @@ static void *_resize_handler(int sig)
if (params.cluster_dims == 4) {
startx = width;
- COLS -= 2;
- width = COLS - width;
+ width = COLS - width - 2;
height = LINES;
} else if (params.cluster_dims == 3) {
startx = width;
- COLS -= 2;
- width = COLS - width;
+ width = COLS - width - 2;
height = LINES;
} else {
startx = 0;
diff --git a/src/squeue/print.c b/src/squeue/print.c
index a0e0bd245e0b39984a2fed5206b6a2f8eda7a35a..b33ad4f8626999f6ebfa50f4e0f1928af9453e6d 100644
--- a/src/squeue/print.c
+++ b/src/squeue/print.c
@@ -202,13 +202,20 @@ static void _combine_pending_array_tasks(List job_list)
bitmap_size = bit_size(task_bitmap);
task_iterator = list_iterator_create(job_list);
while ((task_rec_ptr = list_next(task_iterator))) {
- if (!IS_JOB_PENDING(task_rec_ptr->job_ptr) ||
- (task_rec_ptr == job_rec_ptr) ||
+ if (!IS_JOB_PENDING(task_rec_ptr->job_ptr))
+ continue; /* Not pending */
+ if ((task_rec_ptr == job_rec_ptr) ||
(task_rec_ptr->job_ptr->array_job_id !=
job_rec_ptr->job_ptr->array_job_id) ||
(task_rec_ptr->job_ptr->array_task_id >=
bitmap_size))
- continue;
+ continue; /* Different job array ID */
+ if (xstrcmp(task_rec_ptr->job_ptr->name,
+ job_rec_ptr->job_ptr->name))
+ continue; /* Different name */
+ if (xstrcmp(task_rec_ptr->job_ptr->partition,
+ job_rec_ptr->job_ptr->partition))
+ continue; /* Different partition */
/* Combine this task into master job array record */
update_cnt++;
bit_set(task_bitmap,
@@ -959,6 +966,7 @@ int _print_job_reason_list(job_info_t * job, int width, bool right,
reason = job_reason_string(job->state_reason);
xstrfmtcat(reason_fmt, "(%s)", reason);
_print_str(reason_fmt, width, right, true);
+ xfree(reason_fmt);
} else {
char *nodes = xstrdup(job->nodes);
char *ionodes = NULL;
diff --git a/src/squeue/squeue.c b/src/squeue/squeue.c
index 4b08787e9d5fdf911eb4dbb8f7d11715838f7c98..15b5a8c3078e35d011a5d35580274ba5c360449f 100644
--- a/src/squeue/squeue.c
+++ b/src/squeue/squeue.c
@@ -169,7 +169,8 @@ _get_window_width( void )
static int
_print_job ( bool clear_old )
{
- static job_info_msg_t * old_job_ptr = NULL, * new_job_ptr;
+ static job_info_msg_t *old_job_ptr;
+ job_info_msg_t *new_job_ptr;
int error_code;
uint16_t show_flags = 0;
diff --git a/testsuite/expect/globals b/testsuite/expect/globals
index 98920eb8ada54b019198b6c363006fa7be8435a1..325eec79e747183f5c6c9f7bae6a15ee2df357a1 100755
--- a/testsuite/expect/globals
+++ b/testsuite/expect/globals
@@ -1880,6 +1880,37 @@ proc test_emulated { } {
return $emulated
}
+################################################################
+#
+# Proc: test_killonbadexit
+#
+# Purpose: Determine if KillOnBadExit is configured to be 1.
+#
+#
+# Returns 1 if KillOnBadExit is 1.
+#
+################################################################
+
+proc test_killonbadexit { } {
+ global scontrol bin_bash bin_grep
+
+ log_user 0
+ set killonbadexit 0
+ spawn -noecho $bin_bash -c "exec $scontrol show config | $bin_grep KillOnBadExit"
+ expect {
+ -re "KillOnBadExit *= *1" {
+ set killonbadexit 1
+ exp_continue
+ }
+ eof {
+ wait
+ }
+ }
+ log_user 1
+
+ return $killonbadexit
+}
+
################################################################
#
# Proc: get_cycle_count
diff --git a/testsuite/expect/inc21.21_tests b/testsuite/expect/inc21.21_tests
index 438067b4a5c89d011061a31edc4ca2faf5ef7dcb..5b4f0e944f54cbd9b8faa204059d8c677e9ebffe 100644
--- a/testsuite/expect/inc21.21_tests
+++ b/testsuite/expect/inc21.21_tests
@@ -213,7 +213,7 @@ sleep 10"
set running 0
spawn $squeue -h -o "\%t \%r"
expect {
- -re "PD.Assoc*" {
+ -re "PD." {
incr pending
exp_continue
}
@@ -440,6 +440,9 @@ proc inc21_21_grpwall { test_type limit } {
send_user "\n====== Test $test_type"
send_user "(Within: inc21.21_tests function: inc21_21_grpwall) ======\n"
+ # Wait for old jobs to clean up
+ sleep 2
+
# Since wall is a decayed variable lets reset it to make sure the test
# gets exactly what we would expect.
reset_qos_usage "" $test_qos
diff --git a/testsuite/expect/inc22.1.1 b/testsuite/expect/inc22.1.1
index 050c2c16f211399e3c9f5c813c89c28a54e31870..58a86a099a3935ee1de7eeda3e0c1231e2a43e44 100644
--- a/testsuite/expect/inc22.1.1
+++ b/testsuite/expect/inc22.1.1
@@ -37,7 +37,7 @@
proc inc22_1_1 {} {
- global account1 accounts users cluster_cpus job1_start user1
+ global account1 accounts users cluster_cpus job0_start user1
global node0_down_start node0_cpus node1_cpus cluster sreport
global exit_code wc_key_track wckey1 sql_rem
@@ -53,7 +53,7 @@ proc inc22_1_1 {} {
set end_str [timestamp -format %Y-%m-%dT%X -seconds $period_end]
set reported [expr ($period_end - $period_start) * $cluster_cpus]
set down [expr ($period_end-$node0_down_start) * $node0_cpus]
- set alloc_sec [expr ($period_end-$job1_start) * $node1_cpus]
+ set alloc_sec [expr ($period_end-$job0_start) * $node1_cpus]
set wckey_alloc_sec $alloc_sec
set resv 0
set idle [expr $reported - ($down + $alloc_sec + $resv)]
diff --git a/testsuite/expect/inc22.1.3 b/testsuite/expect/inc22.1.3
index 0b9740a9fc8ed514abbbc7b9aff90fb66a5ab54a..66ffb452b4ef0580a3a39a90d658d36ec286bca2 100644
--- a/testsuite/expect/inc22.1.3
+++ b/testsuite/expect/inc22.1.3
@@ -39,7 +39,7 @@ proc inc22_1_3 { } {
global sreport exit_code node0_down_start node0_down_end node0_cpus
global node1_cpus cluster my_pid accounts users cluster_cpus
- global job1_start job1_end job1_cpus job1_alloc job2_start
+ global job0_start job1_end job1_cpus job1_alloc job2_start
global job2_end job2_cpus job2_alloc job2_elig job3_start
global job3_end job3_cpus job3_alloc job3_elig acct1_alloc acct2_alloc
global acct3_alloc total_alloc wckey1_alloc user1_wckey1_alloc
@@ -59,7 +59,7 @@ proc inc22_1_3 { } {
set reported [expr ($period_end - $period_start) * $cluster_cpus]
set down [expr ($node0_down_end - $node0_down_start) * $node0_cpus]
- set alloc_sec [expr ($job1_end-$job1_start) * $job1_cpus]
+ set alloc_sec [expr ($job1_end-$job0_start) * $job1_cpus]
set alloc_sec [expr $alloc_sec + (($job2_end-$job2_start) * $job2_cpus)]
set alloc_sec [expr $alloc_sec + (($job3_end-$job3_start) * $job3_cpus)]
set wckey_alloc_sec1 [expr $job1_alloc + $job3_alloc]
diff --git a/testsuite/expect/test1.77 b/testsuite/expect/test1.77
index b9011e77770af1d125e895e7af44f832e8f62b69..3ad56b35363bc77bbdc8254fd17c98d2280373ac 100755
--- a/testsuite/expect/test1.77
+++ b/testsuite/expect/test1.77
@@ -38,6 +38,7 @@ set number "\[0-9\]+"
set name "banana"
set my_cmd ./mycmd
set file_in "test$test_id.in"
+set file_out "test$test_id.out"
# Set env path to SLURM_JOB_NAME
set env(SLURM_JOB_NAME) "zebra"
@@ -77,13 +78,11 @@ send_user "SUCCESS\n"
# Tests -J using sbatch
#
set found 0
-set sbatch_pid [spawn $sbatch -J $name --wrap=env]
+set sbatch_pid [spawn $sbatch -J $name -o $file_out --wrap=env]
expect {
-re "Submitted batch job ($number)" {
set job_id $expect_out(1,string)
exp_continue
-
-
}
timeout {
send_user "\nFAILURE: sbatch not responding\n"
@@ -94,8 +93,8 @@ expect {
wait
}
}
-set wait_file [wait_for_file slurm-$job_id.out]
-spawn $bin_grep SLURM_JOB_NAME=banana slurm-$job_id.out
+set wait_file [wait_for_file $file_out]
+spawn $bin_grep SLURM_JOB_NAME=banana $file_out
expect {
-re "SLURM_JOB_NAME=($alpha_numeric)" {
send_user "Job name matched $expect_out(1,string)\n"
@@ -112,7 +111,7 @@ if {$found != 1} {
exit 1
}
if {$exit_code == 0} {
- exec $bin_rm -f slurm-$job_id.out
+ exec $bin_rm -f $file_out
send_user "SUCCESS\n"
}
exec rm -f slurm-$job_id.out
diff --git a/testsuite/expect/test2.15 b/testsuite/expect/test2.15
index 87ad3d6fd8d0fbf67bf0545916c10a15f9ecf98e..9b5b008fcce75035ea10dc830a3d15f246c2e8ab 100755
--- a/testsuite/expect/test2.15
+++ b/testsuite/expect/test2.15
@@ -52,6 +52,9 @@ if { [test_xcpu] } {
} elseif {[test_launch_poe]} {
send_user "\nWARNING: This test is incompatible with POE systems\n"
exit 0
+} elseif {[test_killonbadexit]} {
+ send_user "\nWARNING: This test is incompatible with KillOnBadExit=1\n"
+ exit 0
}
set node_cnt 2
diff --git a/testsuite/expect/test21.21 b/testsuite/expect/test21.21
index 50d9a328b79ef54fccc989cd3769ce20b59bd31c..5508c3342e82b7b6413b180f56ce733742280fcd 100755
--- a/testsuite/expect/test21.21
+++ b/testsuite/expect/test21.21
@@ -83,6 +83,11 @@ array set acct_mod_assoc_test_vals {
print_header $test_id
+if { [string compare [priority_type] multifactor] } {
+ send_user "\nWARNING: test only compatible with priority/multifactor plugin\n"
+ exit $exit_code
+}
+
# Determine what the selecttype param is
if {[test_select_type_params "CR_CORE"]} {
set selectparam 1
diff --git a/testsuite/expect/test22.1 b/testsuite/expect/test22.1
index 1d30b6567ec3a2afc170f80d5ba7dfee26a35be3..652500daa39a9e19cec1089f4119d87e12f6831c 100755
--- a/testsuite/expect/test22.1
+++ b/testsuite/expect/test22.1
@@ -102,17 +102,21 @@ set period_end [exec date -d $end_date +%s]
set start_str [timestamp -format %Y-%m-%dT%X -seconds $period_start]
set end_str [timestamp -format %Y-%m-%dT%X -seconds $period_end]
+#job0 - we really want this to look like job1 but run right before hand.
+set job0_start $period_start
+set job0_run 1200
+set job0_end [expr $job0_start + $job0_run]
#job1
-set job1_start $period_start
-set job1_run 3900
-set job1_end [expr $job1_start+$job1_run]
+set job1_start [expr $period_start + 1200]
+set job1_run 2700
+set job1_end [expr $job1_start + $job1_run]
# This will give us the correct time we ran for
set job1_diff_str [timestamp -format %X -seconds [expr $midnight+$job1_run]]
set job1_start_str [timestamp -format %Y-%m-%dT%X -seconds $job1_start]
set job1_end_str [timestamp -format %Y-%m-%dT%X -seconds $job1_end]
set job1_nodes $node1
set job1_cpus $node1_cpus
-set job1_alloc [expr $job1_run * $job1_cpus]
+set job1_alloc [expr ($job1_run + $job0_run) * $job1_cpus]
set job1_acct $account1
#job2
@@ -141,7 +145,8 @@ set job3_start $job2_end
set job3_run 3900
set job3_end [expr $job3_start+$job3_run]
# This will give us the correct time we ran for
-set job3_diff_str [timestamp -format %X -seconds [expr $midnight+$job1_run]]
+set job3_diff_str [timestamp -format %X -seconds [expr $midnight+$job3_run]]
+
set job3_start_str [timestamp -format %Y-%m-%dT%X -seconds $job3_start]
set job3_end_str [timestamp -format %Y-%m-%dT%X -seconds $job3_end]
#run on just node0
@@ -331,6 +336,7 @@ puts $file "on duplicate key update period_start=VALUES(period_start), period_en
#now we will put in a job running for an hour and 5 minutes
puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, track_steps, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values"
+puts $file "('65536', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job1_acct', $job0_start, $job0_start, $job0_start, $job0_end, '0', 'test_job1', '0', '3', '0', '$job1_cpus', $job1_cpus, '1=$job1_cpus', '$job1_nodes', '0', '0', '0')"
puts $file "('65537', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job1_acct', $job1_start, $job1_start, $job1_start, $job1_end, '0', 'test_job1', '0', '3', '0', '$job1_cpus', $job1_cpus, '1=$job1_cpus', '$job1_nodes', '0', '0', '0')"
puts $file ", ('65538', '$user2acct3', '$wckey1', '$user2wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job2_acct', $job2_elig, $job2_elig, $job2_start, $job2_end, '0', 'test_job2', '0', '3', '0', '$job2_cpus', '$job2_cpus', '1=$job2_cpus', '$job2_nodes', '0', '0', '0')"
puts $file ", ('65539', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job3_acct', $job3_elig, $job3_elig, $job3_start, $job3_end, '0', 'test_job3', '0', '3', '0', '$job3_cpus', '$job3_cpus', '1=$job3_cpus', '$job3_nodes', '0', '0', '0')"
diff --git a/testsuite/expect/test4.13 b/testsuite/expect/test4.13
index a762c40d9074ff23c0272079f0f25c1f70112f5e..b926898065a1f602f80254c0012492445bfd2a11 100755
--- a/testsuite/expect/test4.13
+++ b/testsuite/expect/test4.13
@@ -52,7 +52,7 @@ array set node_info {
FreeMem freemem
Gres gres
NodeAddr nodeaddr
- NodeName nodehost
+ NodeHostName nodehost
RealMemory memory
State statelong
Sockets sockets
@@ -103,8 +103,7 @@ foreach option [array names node_info] {
spawn $scontrol show node $test_node
expect {
-re "\\m$option=($alpha_numeric_dot)\\M" {
- set node_sinfo_vals($node_info($option)) \
- $expect_out(1,string)
+ set node_sinfo_vals($node_info($option)) $expect_out(1,string)
exp_continue
}
timeout {
@@ -122,11 +121,6 @@ set first_option 1
foreach option [array names node_sinfo_vals] {
set match 0
- if {$first_option == 1} {
- set first_option 0
- } else {
- log_user 0
- }
spawn $sinfo -n$test_node -O$option -h
expect {
-nocase -re "$node_sinfo_vals($option)" {
@@ -145,10 +139,9 @@ foreach option [array names node_sinfo_vals] {
if {$match != 1} {
send_user "\nFAILURE: Node information $option output does "
send_user "not match sinfo output. "
- send_user "Looking for value: $node_sinfo_vals($option)\n"
- exit 1
+ send_user "Looking for value: $option = $node_sinfo_vals($option)\n"
+ set exit_code 1
}
- log_user 1
}
#
@@ -218,11 +211,6 @@ set first_option 1
foreach option [array names part_sinfo_vals] {
set match 0
- if {$first_option == 1} {
- set $first_option 0
- } else {
- log_user 0
- }
spawn $sinfo -p$test_part -O$option -h
expect {
-nocase -re "$part_sinfo_vals($option)" {
@@ -243,15 +231,16 @@ foreach option [array names part_sinfo_vals] {
send_user "output does not match sinfo output. "
send_user "Looking for value: "
send_user "$option = $part_sinfo_vals($option)\n"
- cleanup
- exit 1
+ set exit_code 1
}
- log_user 1
}
cleanup
if {$exit_code == 0} {
send_user "\nSUCCESS\n"
+} else {
+ send_user "\nFAILURE\n"
}
+
exit $exit_code