Skip to content

Commit e33ed95

Browse files
mkolodeznymkolodezny
authored
Add option to dispatch activity tasks locally (#564)
1 parent cd2f6bf commit e33ed95

9 files changed

+412
-35
lines changed

‎src/main/java/com/uber/cadence/internal/metrics/MetricsType.java

+11
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package com.uber.cadence.internal.metrics;
1919

2020
public class MetricsType {
21+
2122
public static final String CADENCE_METRICS_PREFIX = "cadence-";
2223
public static final String WORKFLOW_START_COUNTER = CADENCE_METRICS_PREFIX + "workflow-start";
2324
public static final String WORKFLOW_COMPLETED_COUNTER =
@@ -119,6 +120,16 @@ public class MetricsType {
119120
CADENCE_METRICS_PREFIX + "local-activity-panic";
120121
public static final String LOCAL_ACTIVITY_EXECUTION_LATENCY =
121122
CADENCE_METRICS_PREFIX + "local-activity-execution-latency";
123+
public static final String LOCALLY_DISPATCHED_ACTIVITY_POLL_TOTAL_COUNTER =
124+
CADENCE_METRICS_PREFIX + "locally-dispatched-activity-poll-total";
125+
public static final String LOCALLY_DISPATCHED_ACTIVITY_POLL_NO_TASK_COUNTER =
126+
CADENCE_METRICS_PREFIX + "locally-dispatched-activity-poll-no-task";
127+
public static final String LOCALLY_DISPATCHED_ACTIVITY_POLL_SUCCEED_COUNTER =
128+
CADENCE_METRICS_PREFIX + "locally-dispatched-activity-poll-succeed";
129+
public static final String ACTIVITY_LOCAL_DISPATCH_FAILED_COUNTER =
130+
CADENCE_METRICS_PREFIX + "activity-local-dispatch-failed";
131+
public static final String ACTIVITY_LOCAL_DISPATCH_SUCCEED_COUNTER =
132+
CADENCE_METRICS_PREFIX + "activity-local-dispatch-succeed";
122133
public static final String WORKER_PANIC_COUNTER = CADENCE_METRICS_PREFIX + "worker-panic";
123134

124135
public static final String TASK_LIST_QUEUE_LATENCY =

‎src/main/java/com/uber/cadence/internal/sync/SyncWorkflowWorker.java

+42-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.uber.cadence.internal.replay.ReplayDecisionTaskHandler;
2727
import com.uber.cadence.internal.worker.DecisionTaskHandler;
2828
import com.uber.cadence.internal.worker.LocalActivityWorker;
29+
import com.uber.cadence.internal.worker.LocallyDispatchedActivityWorker;
2930
import com.uber.cadence.internal.worker.SingleWorkerOptions;
3031
import com.uber.cadence.internal.worker.SuspendableWorker;
3132
import com.uber.cadence.internal.worker.WorkflowWorker;
@@ -49,10 +50,14 @@ public class SyncWorkflowWorker
4950

5051
private final WorkflowWorker workflowWorker;
5152
private final LocalActivityWorker laWorker;
53+
private final LocallyDispatchedActivityWorker ldaWorker;
54+
5255
private final POJOWorkflowImplementationFactory factory;
5356
private final DataConverter dataConverter;
5457
private final POJOActivityTaskHandler laTaskHandler;
58+
private final POJOActivityTaskHandler ldaTaskHandler;
5559
private final ScheduledExecutorService heartbeatExecutor = Executors.newScheduledThreadPool(4);
60+
private final ScheduledExecutorService ldaHeartbeatExecutor = Executors.newScheduledThreadPool(4);
5661

5762
public SyncWorkflowWorker(
5863
IWorkflowService service,
@@ -61,6 +66,7 @@ public SyncWorkflowWorker(
6166
Function<WorkflowInterceptor, WorkflowInterceptor> interceptorFactory,
6267
SingleWorkerOptions workflowOptions,
6368
SingleWorkerOptions localActivityOptions,
69+
SingleWorkerOptions locallyDispatchedActivityOptions,
6470
DeciderCache cache,
6571
String stickyTaskListName,
6672
Duration stickyDecisionScheduleToStartTimeout,
@@ -91,10 +97,25 @@ public SyncWorkflowWorker(
9197
stickyDecisionScheduleToStartTimeout,
9298
service,
9399
laWorker.getLocalActivityTaskPoller());
100+
ldaTaskHandler =
101+
new POJOActivityTaskHandler(
102+
service,
103+
domain,
104+
locallyDispatchedActivityOptions.getDataConverter(),
105+
ldaHeartbeatExecutor);
106+
ldaWorker =
107+
new LocallyDispatchedActivityWorker(
108+
service, domain, taskList, locallyDispatchedActivityOptions, ldaTaskHandler);
94109

95110
workflowWorker =
96111
new WorkflowWorker(
97-
service, domain, taskList, workflowOptions, taskHandler, stickyTaskListName);
112+
service,
113+
domain,
114+
taskList,
115+
workflowOptions,
116+
taskHandler,
117+
ldaWorker.getLocallyDispatchedActivityTaskPoller(),
118+
stickyTaskListName);
98119
}
99120

100121
public void setWorkflowImplementationTypes(
@@ -115,64 +136,80 @@ public void setLocalActivitiesImplementation(Object... activitiesImplementation)
115136
this.laTaskHandler.setLocalActivitiesImplementation(activitiesImplementation);
116137
}
117138

139+
public void setActivitiesImplementationToDispatchLocally(Object... activitiesImplementation) {
140+
this.ldaTaskHandler.setActivitiesImplementation(activitiesImplementation);
141+
}
142+
118143
@Override
119144
public void start() {
120145
workflowWorker.start();
121146
// workflowWorker doesn't start if no types are registered with it. In that case we don't need
122147
// to start LocalActivity Worker.
123148
if (workflowWorker.isStarted()) {
124149
laWorker.start();
150+
ldaWorker.start();
125151
}
126152
}
127153

128154
@Override
129155
public boolean isStarted() {
130-
return workflowWorker.isStarted() && laWorker.isStarted();
156+
return workflowWorker.isStarted() && laWorker.isStarted() && ldaWorker.isStarted();
131157
}
132158

133159
@Override
134160
public boolean isShutdown() {
135-
return workflowWorker.isShutdown() && laWorker.isShutdown();
161+
return workflowWorker.isShutdown() && laWorker.isShutdown() && ldaWorker.isShutdown();
136162
}
137163

138164
@Override
139165
public boolean isTerminated() {
140-
return workflowWorker.isTerminated() && laWorker.isTerminated();
166+
return workflowWorker.isTerminated()
167+
&& laWorker.isTerminated()
168+
&& ldaHeartbeatExecutor.isTerminated()
169+
&& ldaWorker.isTerminated();
141170
}
142171

143172
@Override
144173
public void shutdown() {
145174
laWorker.shutdown();
175+
ldaHeartbeatExecutor.shutdown();
176+
ldaWorker.shutdown();
146177
workflowWorker.shutdown();
147178
}
148179

149180
@Override
150181
public void shutdownNow() {
151182
laWorker.shutdownNow();
183+
ldaHeartbeatExecutor.shutdownNow();
184+
ldaWorker.shutdownNow();
152185
workflowWorker.shutdownNow();
153186
}
154187

155188
@Override
156189
public void awaitTermination(long timeout, TimeUnit unit) {
157190
long timeoutMillis = InternalUtils.awaitTermination(laWorker, unit.toMillis(timeout));
191+
timeoutMillis = InternalUtils.awaitTermination(ldaHeartbeatExecutor, timeoutMillis);
192+
timeoutMillis = InternalUtils.awaitTermination(ldaWorker, timeoutMillis);
158193
InternalUtils.awaitTermination(workflowWorker, timeoutMillis);
159194
}
160195

161196
@Override
162197
public void suspendPolling() {
163198
workflowWorker.suspendPolling();
164199
laWorker.suspendPolling();
200+
ldaWorker.suspendPolling();
165201
}
166202

167203
@Override
168204
public void resumePolling() {
169205
workflowWorker.resumePolling();
170206
laWorker.resumePolling();
207+
ldaWorker.resumePolling();
171208
}
172209

173210
@Override
174211
public boolean isSuspended() {
175-
return workflowWorker.isSuspended() && laWorker.isSuspended();
212+
return workflowWorker.isSuspended() && laWorker.isSuspended() && ldaWorker.isSuspended();
176213
}
177214

178215
public <R> R queryWorkflowExecution(

‎src/main/java/com/uber/cadence/internal/worker/ActivityPollTask.java

+4-16
Original file line numberDiff line numberDiff line change
@@ -26,33 +26,29 @@
2626
import com.uber.cadence.internal.metrics.MetricsType;
2727
import com.uber.cadence.serviceclient.IWorkflowService;
2828
import com.uber.m3.tally.Stopwatch;
29-
import com.uber.m3.util.Duration;
3029
import org.apache.thrift.TException;
3130
import org.slf4j.Logger;
3231
import org.slf4j.LoggerFactory;
3332

34-
final class ActivityPollTask implements Poller.PollTask<PollForActivityTaskResponse> {
33+
final class ActivityPollTask extends ActivityPollTaskBase {
3534

35+
private static final Logger log = LoggerFactory.getLogger(ActivityPollTask.class);
3636
private final IWorkflowService service;
3737
private final String domain;
3838
private final String taskList;
39-
private final SingleWorkerOptions options;
40-
private static final Logger log = LoggerFactory.getLogger(ActivityPollTask.class);
4139

4240
public ActivityPollTask(
4341
IWorkflowService service, String domain, String taskList, SingleWorkerOptions options) {
44-
42+
super(options);
4543
this.service = service;
4644
this.domain = domain;
4745
this.taskList = taskList;
48-
this.options = options;
4946
}
5047

5148
@Override
52-
public PollForActivityTaskResponse poll() throws TException {
49+
protected PollForActivityTaskResponse pollTask() throws TException {
5350
options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_COUNTER).inc(1);
5451
Stopwatch sw = options.getMetricsScope().timer(MetricsType.ACTIVITY_POLL_LATENCY).start();
55-
5652
PollForActivityTaskRequest pollRequest = new PollForActivityTaskRequest();
5753
pollRequest.setDomain(domain);
5854
pollRequest.setIdentity(options.getIdentity());
@@ -89,14 +85,6 @@ public PollForActivityTaskResponse poll() throws TException {
8985
if (log.isTraceEnabled()) {
9086
log.trace("poll request returned " + result);
9187
}
92-
93-
options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_SUCCEED_COUNTER).inc(1);
94-
options
95-
.getMetricsScope()
96-
.timer(MetricsType.ACTIVITY_SCHEDULED_TO_START_LATENCY)
97-
.record(
98-
Duration.ofNanos(
99-
result.getStartedTimestamp() - result.getScheduledTimestampOfThisAttempt()));
10088
sw.stop();
10189
return result;
10290
}

‎src/main/java/com/uber/cadence/internal/worker/ActivityPollTaskBase.java

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Modifications copyright (C) 2017 Uber Technologies, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not
7+
* use this file except in compliance with the License. A copy of the License is
8+
* located at
9+
*
10+
* http://aws.amazon.com/apache2.0
11+
*
12+
* or in the "license" file accompanying this file. This file is distributed on
13+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
14+
* express or implied. See the License for the specific language governing
15+
* permissions and limitations under the License.
16+
*/
17+
18+
package com.uber.cadence.internal.worker;
19+
20+
import com.uber.cadence.PollForActivityTaskResponse;
21+
import com.uber.cadence.internal.metrics.MetricsType;
22+
import com.uber.m3.util.Duration;
23+
import org.apache.thrift.TException;
24+
25+
abstract class ActivityPollTaskBase implements Poller.PollTask<PollForActivityTaskResponse> {
26+
27+
protected final SingleWorkerOptions options;
28+
29+
public ActivityPollTaskBase(SingleWorkerOptions options) {
30+
this.options = options;
31+
}
32+
33+
public PollForActivityTaskResponse poll() throws TException {
34+
35+
PollForActivityTaskResponse result = pollTask();
36+
if (result == null || result.getTaskToken() == null) {
37+
return null;
38+
}
39+
options.getMetricsScope().counter(MetricsType.ACTIVITY_POLL_SUCCEED_COUNTER).inc(1);
40+
options
41+
.getMetricsScope()
42+
.timer(MetricsType.ACTIVITY_SCHEDULED_TO_START_LATENCY)
43+
.record(
44+
Duration.ofNanos(
45+
result.getStartedTimestamp() - result.getScheduledTimestampOfThisAttempt()));
46+
return result;
47+
}
48+
49+
protected abstract PollForActivityTaskResponse pollTask() throws TException;
50+
}

‎src/main/java/com/uber/cadence/internal/worker/ActivityWorker.java

+19-6
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.uber.cadence.internal.metrics.MetricsTag;
3030
import com.uber.cadence.internal.metrics.MetricsType;
3131
import com.uber.cadence.internal.worker.ActivityTaskHandler.Result;
32+
import com.uber.cadence.internal.worker.Poller.PollTask;
3233
import com.uber.cadence.serviceclient.IWorkflowService;
3334
import com.uber.m3.tally.Scope;
3435
import com.uber.m3.tally.Stopwatch;
@@ -43,22 +44,30 @@
4344
import org.apache.thrift.TException;
4445
import org.slf4j.MDC;
4546

46-
public final class ActivityWorker extends SuspendableWorkerBase {
47-
48-
private static final String POLL_THREAD_NAME_PREFIX = "Activity Poller taskList=";
47+
public class ActivityWorker extends SuspendableWorkerBase {
4948

49+
protected final SingleWorkerOptions options;
5050
private final ActivityTaskHandler handler;
5151
private final IWorkflowService service;
5252
private final String domain;
5353
private final String taskList;
54-
private final SingleWorkerOptions options;
5554

5655
public ActivityWorker(
5756
IWorkflowService service,
5857
String domain,
5958
String taskList,
6059
SingleWorkerOptions options,
6160
ActivityTaskHandler handler) {
61+
this(service, domain, taskList, options, handler, "Activity Poller taskList=");
62+
}
63+
64+
public ActivityWorker(
65+
IWorkflowService service,
66+
String domain,
67+
String taskList,
68+
SingleWorkerOptions options,
69+
ActivityTaskHandler handler,
70+
String pollThreadNamePrefix) {
6271
this.service = Objects.requireNonNull(service);
6372
this.domain = Objects.requireNonNull(domain);
6473
this.taskList = Objects.requireNonNull(taskList);
@@ -69,7 +78,7 @@ public ActivityWorker(
6978
pollerOptions =
7079
PollerOptions.newBuilder(pollerOptions)
7180
.setPollThreadNamePrefix(
72-
POLL_THREAD_NAME_PREFIX + "\"" + taskList + "\", domain=\"" + domain + "\"")
81+
pollThreadNamePrefix + "\"" + taskList + "\", domain=\"" + domain + "\"")
7382
.build();
7483
}
7584
this.options = SingleWorkerOptions.newBuilder(options).setPollerOptions(pollerOptions).build();
@@ -81,7 +90,7 @@ public void start() {
8190
SuspendableWorker poller =
8291
new Poller<>(
8392
options.getIdentity(),
84-
new ActivityPollTask(service, domain, taskList, options),
93+
getOrCreateActivityPollTask(),
8594
new PollTaskExecutor<>(domain, taskList, options, new TaskHandlerImpl(handler)),
8695
options.getPollerOptions(),
8796
options.getMetricsScope());
@@ -91,6 +100,10 @@ public void start() {
91100
}
92101
}
93102

103+
protected PollTask<PollForActivityTaskResponse> getOrCreateActivityPollTask() {
104+
return new ActivityPollTask(service, domain, taskList, options);
105+
}
106+
94107
private class TaskHandlerImpl
95108
implements PollTaskExecutor.TaskHandler<PollForActivityTaskResponse> {
96109

0 commit comments

Comments
 (0)