Merge branch 'master' into clif/add_binoculars_to_python_client

internal/scheduler/submitcheck.go

@@ -186,16 +186,13 @@ func (srv *SubmitChecker) getIndividualSchedulingResult(jctx *context.JobSchedul
 // TODO: there are a number of things this won't catch:
 //   - Node Uniformity Label (although it will work if this is per cluster)
 //   - Gang jobs that will use more than the allowed capacity limit
-func (srv *SubmitChecker) getSchedulingResult(gctx *context.GangSchedulingContext, state *schedulerState) schedulingResult {
+func (srv *SubmitChecker) getSchedulingResult(originalGangCtx *context.GangSchedulingContext, state *schedulerState) schedulingResult {
 	sucessfulPools := map[string]bool{}
 	var sb strings.Builder
 
 poolStart:
 	for _, pool := range srv.schedulingConfig.Pools {
 
-		// copy the gctx here, as we are going to mutate it
-		gctx = copyGangContext(gctx)
-
 		if sucessfulPools[pool.Name] {
 			continue
 		}
@@ -212,6 +209,10 @@ poolStart:
 		}
 
 		for _, ex := range executors {
+
+			// copy the gctx here, as we are going to mutate it
+			gctx := copyGangContext(originalGangCtx)
+
 			txn := ex.nodeDb.Txn(true)
 			ok, err := ex.nodeDb.ScheduleManyWithTxn(txn, gctx)
 			txn.Abort()

internal/scheduler/submitcheck_test.go

@@ -105,6 +105,18 @@ func TestSubmitChecker_CheckJobDbJobs(t *testing.T) {
 				smallAwayJob.Id(): {isSchedulable: true, pools: []string{"cpu", "cpu-away"}},
 			},
 		},
+		"One job schedulable, away pools, multiple executors": {
+			executorTimout: defaultTimeout,
+			executors: []*schedulerobjects.Executor{
+				Executor(SmallNode("cpu")),
+				Executor(GpuNode("gpu")),
+				Executor(GpuNode("gpu")),
+			},
+			jobs: []*jobdb.Job{smallAwayJob},
+			expectedResult: map[string]schedulingResult{
+				smallAwayJob.Id(): {isSchedulable: true, pools: []string{"cpu", "cpu-away"}},
+			},
+		},
 		"One job schedulable, multiple pools": {
 			executorTimout: defaultTimeout,
 			executors: []*schedulerobjects.Executor{