Skip to content

Commit b1b1221

Browse files
haitchhaitch
authored
Merge pull request #7 from Azure/haitao/job-input
simplify job with interface
2 parents ac85f3a + b6f55e6 commit b1b1221

File tree

2 files changed

+250
-220
lines changed

2 files changed

+250
-220
lines changed

‎job.go

+29-220
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,17 @@ const JobStatePending JobState = "pending"
1616
const JobStateRunning JobState = "running"
1717
const JobStateCompleted JobState = "completed"
1818

19-
const rootStepName = "job"
19+
const rootStepName = "$job"
20+
21+
type JobInterface interface {
22+
GetStep(stepName string) (StepMeta, bool) // switch bool to error
23+
AddStep(step StepMeta, precedingSteps ...StepMeta)
24+
RootStep() *StepInfo[interface{}]
25+
26+
Start(ctx context.Context) error
27+
28+
RuntimeContext() context.Context
29+
}
2030

2131
type Job struct {
2232
Name string
@@ -32,6 +42,8 @@ type Job struct {
3242
runtimeCtx context.Context
3343
}
3444

45+
var _ JobInterface = &Job{}
46+
3547
func NewJob(name string) *Job {
3648
jobStart := sync.WaitGroup{}
3749
jobStart.Add(1)
@@ -64,220 +76,23 @@ func NewJob(name string) *Job {
6476
return j
6577
}
6678

67-
func InputParam[T any](bCtx context.Context, j *Job, stepName string, value *T) *StepInfo[T] {
68-
step := newStepInfo[T](stepName, stepTypeParam)
69-
70-
instrumentedFunc := func(ctx context.Context) (*T, error) {
71-
j.rootStep.Wait(ctx)
72-
step.executionData.StartTime = time.Now()
73-
step.state = StepStateCompleted
74-
step.executionData.Duration = time.Since(j.rootStep.executionData.StartTime)
75-
return value, nil
76-
}
77-
step.task = asynctask.Start(bCtx, instrumentedFunc)
78-
79-
j.Steps[stepName] = step
80-
j.registerStepInGraph(step, j.rootStep)
81-
82-
return step
79+
func (j *Job) RootStep() *StepInfo[interface{}] {
80+
return j.rootStep
8381
}
8482

85-
func AddStep[T any](bCtx context.Context, j *Job, stepName string, stepFunc asynctask.AsyncFunc[T], optionDecorators ...ExecutionOptionPreparer) (*StepInfo[T], error) {
86-
step := newStepInfo[T](stepName, stepTypeTask, optionDecorators...)
87-
88-
// also consider specified the dependencies from ExecutionOptionPreparer, without consume the result.
89-
var precedingTasks []asynctask.Waitable
90-
var precedingSteps []StepMeta
91-
for _, depStepName := range step.DependsOn() {
92-
if depStep, ok := j.Steps[depStepName]; ok {
93-
precedingTasks = append(precedingTasks, depStep.Waitable())
94-
precedingSteps = append(precedingSteps, depStep)
95-
} else {
96-
return nil, fmt.Errorf("step [%s] not found", depStepName)
97-
}
98-
}
99-
100-
// if a step have no preceding tasks, link it to our rootJob as preceding task, so it won't start yet.
101-
if len(precedingTasks) == 0 {
102-
precedingSteps = append(precedingSteps, j.rootStep)
103-
precedingTasks = append(precedingTasks, j.rootStep.Waitable())
104-
}
105-
106-
// instrument to :
107-
// replaceRuntimeContext,
108-
// trackStepState
109-
// retryHandling (TODO)
110-
// errorHandling (TODO)
111-
// timeoutHandling (TODO)
112-
instrumentedFunc := func(ctx context.Context) (*T, error) {
113-
if err := asynctask.WaitAll(ctx, &asynctask.WaitAllOptions{}, precedingTasks...); err != nil {
114-
/* this only work on ExecuteAfter from input, asynctask.ContinueWith and asynctask.AfterBoth won't invoke instrumentedFunc if any of the preceding task failed.
115-
we need to be consistent on how to set state of dependent step.
116-
step.executionData.StartTime = time.Now()
117-
step.state = StepStateFailed
118-
step.executionData.Duration = 0 */
119-
return nil, newJobError(ErrPrecedentStepFailure, "")
120-
}
121-
step.executionData.StartTime = time.Now()
122-
step.state = StepStateRunning
123-
124-
var result *T
125-
var err error
126-
if step.executionOptions.RetryPolicy != nil {
127-
step.executionData.Retried = &RetryReport{}
128-
result, err = newRetryer(step.executionOptions.RetryPolicy, step.executionData.Retried, func() (*T, error) { return stepFunc(j.runtimeCtx) }).Run()
129-
} else {
130-
result, err = stepFunc(j.runtimeCtx)
131-
}
132-
133-
if err != nil {
134-
step.state = StepStateFailed
135-
} else {
136-
step.state = StepStateCompleted
137-
}
138-
139-
step.executionData.Duration = time.Since(step.executionData.StartTime)
140-
return result, newStepError(stepName, err)
141-
}
142-
143-
step.task = asynctask.Start(bCtx, instrumentedFunc)
144-
145-
j.Steps[stepName] = step
146-
j.registerStepInGraph(step, precedingSteps...)
147-
148-
return step, nil
83+
func (j *Job) GetStep(stepName string) (StepMeta, bool) {
84+
stepMeta, ok := j.Steps[stepName]
85+
return stepMeta, ok
14986
}
15087

151-
func StepAfter[T, S any](bCtx context.Context, j *Job, stepName string, parentStep *StepInfo[T], stepFunc asynctask.ContinueFunc[T, S], optionDecorators ...ExecutionOptionPreparer) (*StepInfo[S], error) {
152-
// check parentStepT is in this job
153-
if get, ok := j.Steps[parentStep.GetName()]; !ok || get != parentStep {
154-
return nil, fmt.Errorf("step [%s] not found in job", parentStep.GetName())
155-
}
156-
157-
step := newStepInfo[S](stepName, stepTypeTask, append(optionDecorators, ExecuteAfter(parentStep))...)
158-
159-
// also consider specified the dependencies from ExecutionOptionPreparer, without consume the result.
160-
var precedingSteps []StepMeta
161-
var precedingTasks []asynctask.Waitable
162-
for _, depStepName := range step.DependsOn() {
163-
if depStep, ok := j.Steps[depStepName]; ok {
164-
precedingSteps = append(precedingSteps, depStep)
165-
precedingTasks = append(precedingTasks, depStep.Waitable())
166-
} else {
167-
return nil, fmt.Errorf("step [%s] not found", depStepName)
168-
}
169-
}
170-
171-
// instrument to :
172-
// replaceRuntimeContext
173-
// trackStepState
174-
// retryHandling (TODO)
175-
// errorHandling (TODO)
176-
// timeoutHandling (TODO)
177-
instrumentedFunc := func(ctx context.Context, t *T) (*S, error) {
178-
if err := asynctask.WaitAll(ctx, &asynctask.WaitAllOptions{}, precedingTasks...); err != nil {
179-
/* this only work on ExecuteAfter from input, asynctask.ContinueWith and asynctask.AfterBoth won't invoke instrumentedFunc if any of the preceding task failed.
180-
we need to be consistent on how to set state of dependent step.
181-
step.executionData.StartTime = time.Now()
182-
step.state = StepStateFailed
183-
step.executionData.Duration = 0 */
184-
return nil, newJobError(ErrPrecedentStepFailure, "")
185-
}
186-
step.executionData.StartTime = time.Now()
187-
step.state = StepStateRunning
188-
var result *S
189-
var err error
190-
if step.executionOptions.RetryPolicy != nil {
191-
step.executionData.Retried = &RetryReport{}
192-
result, err = newRetryer(step.executionOptions.RetryPolicy, step.executionData.Retried, func() (*S, error) { return stepFunc(j.runtimeCtx, t) }).Run()
193-
} else {
194-
result, err = stepFunc(j.runtimeCtx, t)
195-
}
196-
197-
if err != nil {
198-
step.state = StepStateFailed
199-
} else {
200-
step.state = StepStateCompleted
201-
}
202-
203-
step.executionData.Duration = time.Since(step.executionData.StartTime)
204-
return result, newStepError(stepName, err)
205-
}
206-
207-
step.task = asynctask.ContinueWith(bCtx, parentStep.task, instrumentedFunc)
208-
209-
j.Steps[stepName] = step
210-
j.registerStepInGraph(step, precedingSteps...)
211-
return step, nil
212-
}
213-
214-
func StepAfterBoth[T, S, R any](bCtx context.Context, j *Job, stepName string, parentStepT *StepInfo[T], parentStepS *StepInfo[S], stepFunc asynctask.AfterBothFunc[T, S, R], optionDecorators ...ExecutionOptionPreparer) (*StepInfo[R], error) {
215-
// check parentStepT is in this job
216-
if get, ok := j.Steps[parentStepT.GetName()]; !ok || get != parentStepT {
217-
return nil, fmt.Errorf("step [%s] not found in job", parentStepT.GetName())
218-
}
219-
if get, ok := j.Steps[parentStepS.GetName()]; !ok || get != parentStepS {
220-
return nil, fmt.Errorf("step [%s] not found in job", parentStepS.GetName())
221-
}
222-
223-
step := newStepInfo[R](stepName, stepTypeTask, append(optionDecorators, ExecuteAfter(parentStepT), ExecuteAfter(parentStepS))...)
224-
225-
// also consider specified the dependencies from ExecutionOptionPreparer, without consume the result.
226-
var precedingSteps []StepMeta
227-
var precedingTasks []asynctask.Waitable
228-
for _, depStepName := range step.DependsOn() {
229-
if depStep, ok := j.Steps[depStepName]; ok {
230-
precedingSteps = append(precedingSteps, depStep)
231-
precedingTasks = append(precedingTasks, depStep.Waitable())
232-
} else {
233-
return nil, fmt.Errorf("step [%s] not found", depStepName)
234-
}
235-
}
236-
237-
// instrument to :
238-
// replaceRuntimeContext
239-
// trackStepState
240-
// retryHandling (TODO)
241-
// errorHandling (TODO)
242-
// timeoutHandling (TODO)
243-
instrumentedFunc := func(ctx context.Context, t *T, s *S) (*R, error) {
244-
if err := asynctask.WaitAll(ctx, &asynctask.WaitAllOptions{}, precedingTasks...); err != nil {
245-
/* this only work on ExecuteAfter from input, asynctask.ContinueWith and asynctask.AfterBoth won't invoke instrumentedFunc if any of the preceding task failed.
246-
we need to be consistent on how to set state of dependent step.
247-
step.executionData.StartTime = time.Now()
248-
step.state = StepStateFailed
249-
step.executionData.Duration = 0 */
250-
return nil, newJobError(ErrPrecedentStepFailure, "")
251-
}
252-
253-
step.executionData.StartTime = time.Now()
254-
step.state = StepStateRunning
255-
256-
var result *R
257-
var err error
258-
if step.executionOptions.RetryPolicy != nil {
259-
step.executionData.Retried = &RetryReport{}
260-
result, err = newRetryer(step.executionOptions.RetryPolicy, step.executionData.Retried, func() (*R, error) { return stepFunc(j.runtimeCtx, t, s) }).Run()
261-
} else {
262-
result, err = stepFunc(j.runtimeCtx, t, s)
263-
}
264-
265-
if err != nil {
266-
step.state = StepStateFailed
267-
} else {
268-
step.state = StepStateCompleted
269-
}
270-
271-
step.executionData.Duration = time.Since(step.executionData.StartTime)
272-
return result, newStepError(stepName, err)
88+
func (j *Job) AddStep(step StepMeta, precedingSteps ...StepMeta) {
89+
// TODO: check conflict
90+
j.Steps[step.GetName()] = step
91+
stepNode := newStepNode(step)
92+
j.stepsDag.AddNode(stepNode)
93+
for _, precedingStep := range precedingSteps {
94+
j.stepsDag.Connect(precedingStep.getID(), step.getID())
27395
}
274-
275-
step.task = asynctask.AfterBoth(bCtx, parentStepT.task, parentStepS.task, instrumentedFunc)
276-
277-
j.Steps[stepName] = step
278-
j.registerStepInGraph(step, precedingSteps...)
279-
280-
return step, nil
28196
}
28297

28398
func (j *Job) Start(ctx context.Context) error {
@@ -290,6 +105,10 @@ func (j *Job) Start(ctx context.Context) error {
290105
return nil
291106
}
292107

108+
func (j *Job) RuntimeContext() context.Context {
109+
return j.runtimeCtx
110+
}
111+
293112
func (j *Job) Wait(ctx context.Context) error {
294113
var tasks []asynctask.Waitable
295114
for _, step := range j.Steps {
@@ -298,16 +117,6 @@ func (j *Job) Wait(ctx context.Context) error {
298117
return asynctask.WaitAll(ctx, &asynctask.WaitAllOptions{}, tasks...)
299118
}
300119

301-
func (j *Job) registerStepInGraph(step StepMeta, precedingSteps ...StepMeta) error {
302-
stepNode := newStepNode(step)
303-
j.stepsDag.AddNode(stepNode)
304-
for _, precedingStep := range precedingSteps {
305-
j.stepsDag.Connect(precedingStep.getID(), step.getID())
306-
}
307-
308-
return nil
309-
}
310-
311120
// Visualize return a DAG of the job execution graph
312121
func (j *Job) Visualize() (string, error) {
313122
return j.stepsDag.ToDotGraph()

0 commit comments

Comments
 (0)