Skip to content

Commit 4c23ac5

Browse files
anandamideShakyananandamideShakyan
committed
[native] Differentiate number of drivers and number of splits in stats reporting (Part-1)
1 parent 9d1cc51 commit 4c23ac5

26 files changed

+467
-11
lines changed

‎presto-main/src/main/java/com/facebook/presto/execution/BasicStageExecutionStats.java

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ public class BasicStageExecutionStats
3434
public static final BasicStageExecutionStats EMPTY_STAGE_STATS = new BasicStageExecutionStats(
3535
false,
3636

37+
0,
38+
0,
39+
0,
40+
0,
3741
0,
3842
0,
3943
0,
@@ -62,6 +66,10 @@ public class BasicStageExecutionStats
6266
private final int queuedDrivers;
6367
private final int runningDrivers;
6468
private final int completedDrivers;
69+
private final int totalSplits;
70+
private final int queuedSplits;
71+
private final int runningSplits;
72+
private final int completedSplits;
6573
private final DataSize rawInputDataSize;
6674
private final long rawInputPositions;
6775
private final double cumulativeUserMemory;
@@ -83,6 +91,11 @@ public BasicStageExecutionStats(
8391
int runningDrivers,
8492
int completedDrivers,
8593

94+
int totalSplits,
95+
int queuedSplits,
96+
int runningSplits,
97+
int completedSplits,
98+
8699
DataSize rawInputDataSize,
87100
long rawInputPositions,
88101

@@ -106,6 +119,10 @@ public BasicStageExecutionStats(
106119
this.queuedDrivers = queuedDrivers;
107120
this.runningDrivers = runningDrivers;
108121
this.completedDrivers = completedDrivers;
122+
this.totalSplits = totalSplits;
123+
this.queuedSplits = queuedSplits;
124+
this.runningSplits = runningSplits;
125+
this.completedSplits = completedSplits;
109126
this.rawInputDataSize = requireNonNull(rawInputDataSize, "rawInputDataSize is null");
110127
this.rawInputPositions = rawInputPositions;
111128
this.cumulativeUserMemory = cumulativeUserMemory;
@@ -145,6 +162,26 @@ public int getCompletedDrivers()
145162
return completedDrivers;
146163
}
147164

165+
public int getTotalSplits()
166+
{
167+
return totalSplits;
168+
}
169+
170+
public int getQueuedSplits()
171+
{
172+
return queuedSplits;
173+
}
174+
175+
public int getRunningSplits()
176+
{
177+
return runningSplits;
178+
}
179+
180+
public int getCompletedSplits()
181+
{
182+
return completedSplits;
183+
}
184+
148185
public DataSize getRawInputDataSize()
149186
{
150187
return rawInputDataSize;
@@ -212,6 +249,11 @@ public static BasicStageExecutionStats aggregateBasicStageStats(Iterable<BasicSt
212249
int runningDrivers = 0;
213250
int completedDrivers = 0;
214251

252+
int totalSplits = 0;
253+
int queuedSplits = 0;
254+
int runningSplits = 0;
255+
int completedSplits = 0;
256+
215257
double cumulativeUserMemory = 0;
216258
double cumulativeTotalMemory = 0;
217259
long userMemoryReservation = 0;
@@ -236,6 +278,11 @@ public static BasicStageExecutionStats aggregateBasicStageStats(Iterable<BasicSt
236278
runningDrivers += stageStats.getRunningDrivers();
237279
completedDrivers += stageStats.getCompletedDrivers();
238280

281+
totalSplits += stageStats.getTotalSplits();
282+
queuedSplits += stageStats.getQueuedSplits();
283+
runningSplits += stageStats.getRunningSplits();
284+
completedSplits += stageStats.getCompletedSplits();
285+
239286
cumulativeUserMemory += stageStats.getCumulativeUserMemory();
240287
cumulativeTotalMemory += stageStats.getCumulativeTotalMemory();
241288
userMemoryReservation += stageStats.getUserMemoryReservation().toBytes();
@@ -268,6 +315,11 @@ public static BasicStageExecutionStats aggregateBasicStageStats(Iterable<BasicSt
268315
runningDrivers,
269316
completedDrivers,
270317

318+
totalSplits,
319+
queuedSplits,
320+
runningSplits,
321+
completedSplits,
322+
271323
succinctBytes(rawInputDataSize),
272324
rawInputPositions,
273325

‎presto-main/src/main/java/com/facebook/presto/execution/QueryStateMachine.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,11 @@ public BasicQueryInfo getBasicQueryInfo(Optional<BasicStageExecutionStats> rootS
389389
stageStats.getRunningDrivers(),
390390
stageStats.getCompletedDrivers(),
391391

392+
stageStats.getTotalSplits(),
393+
stageStats.getQueuedSplits(),
394+
stageStats.getRunningSplits(),
395+
stageStats.getCompletedSplits(),
396+
392397
stageStats.getRawInputDataSize(),
393398
stageStats.getRawInputPositions(),
394399

@@ -1302,6 +1307,10 @@ private static QueryStats pruneQueryStats(QueryStats queryStats)
13021307
queryStats.getRunningDrivers(),
13031308
queryStats.getBlockedDrivers(),
13041309
queryStats.getCompletedDrivers(),
1310+
queryStats.getTotalSplits(),
1311+
queryStats.getQueuedSplits(),
1312+
queryStats.getRunningSplits(),
1313+
queryStats.getCompletedSplits(),
13051314
queryStats.getCumulativeUserMemory(),
13061315
queryStats.getCumulativeTotalMemory(),
13071316
queryStats.getUserMemoryReservation(),

‎presto-main/src/main/java/com/facebook/presto/execution/QueryStats.java

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ public class QueryStats
7878
private final int blockedDrivers;
7979
private final int completedDrivers;
8080

81+
private final int totalSplits;
82+
private final int queuedSplits;
83+
private final int runningSplits;
84+
private final int completedSplits;
85+
8186
private final double cumulativeUserMemory;
8287
private final double cumulativeTotalMemory;
8388
private final DataSize userMemoryReservation;
@@ -153,6 +158,11 @@ public QueryStats(
153158
@JsonProperty("blockedDrivers") int blockedDrivers,
154159
@JsonProperty("completedDrivers") int completedDrivers,
155160

161+
@JsonProperty("totalSplits") int totalSplits,
162+
@JsonProperty("queuedSplits") int queuedSplits,
163+
@JsonProperty("runningSplits") int runningSplits,
164+
@JsonProperty("completedSplits") int completedSplits,
165+
156166
@JsonProperty("cumulativeUserMemory") double cumulativeUserMemory,
157167
@JsonProperty("cumulativeTotalMemory") double cumulativeTotalMemory,
158168
@JsonProperty("userMemoryReservation") DataSize userMemoryReservation,
@@ -233,6 +243,14 @@ public QueryStats(
233243
this.blockedDrivers = blockedDrivers;
234244
checkArgument(completedDrivers >= 0, "completedDrivers is negative");
235245
this.completedDrivers = completedDrivers;
246+
checkArgument(totalSplits >= 0, "totalSplits is negative");
247+
this.totalSplits = totalSplits;
248+
checkArgument(queuedSplits >= 0, "queuedSplits is negative");
249+
this.queuedSplits = queuedSplits;
250+
checkArgument(runningSplits >= 0, "runningSplits is negative");
251+
this.runningSplits = runningSplits;
252+
checkArgument(completedSplits >= 0, "completedSplits is negative");
253+
this.completedSplits = completedSplits;
236254
checkArgument(cumulativeUserMemory >= 0, "cumulativeUserMemory is negative");
237255
this.cumulativeUserMemory = cumulativeUserMemory;
238256
checkArgument(cumulativeTotalMemory >= 0, "cumulativeTotalMemory is negative");
@@ -305,6 +323,11 @@ public static QueryStats create(
305323
int blockedDrivers = 0;
306324
int completedDrivers = 0;
307325

326+
int totalSplits = 0;
327+
int queuedSplits = 0;
328+
int runningSplits = 0;
329+
int completedSplits = 0;
330+
308331
double cumulativeUserMemory = 0;
309332
double cumulativeTotalMemory = 0;
310333
long userMemoryReservation = 0;
@@ -354,6 +377,11 @@ public static QueryStats create(
354377
blockedDrivers += stageExecutionStats.getBlockedDrivers();
355378
completedDrivers += stageExecutionStats.getCompletedDrivers();
356379

380+
totalSplits += stageExecutionStats.getTotalSplits();
381+
queuedSplits += stageExecutionStats.getQueuedSplits();
382+
runningSplits += stageExecutionStats.getRunningSplits();
383+
completedSplits += stageExecutionStats.getCompletedSplits();
384+
357385
cumulativeUserMemory += stageExecutionStats.getCumulativeUserMemory();
358386
cumulativeTotalMemory += stageExecutionStats.getCumulativeTotalMemory();
359387
userMemoryReservation += stageExecutionStats.getUserMemoryReservation().toBytes();
@@ -453,6 +481,11 @@ else if (operatorType.equals(TableScanOperator.class.getSimpleName()) || operato
453481
blockedDrivers,
454482
completedDrivers,
455483

484+
totalSplits,
485+
queuedSplits,
486+
runningSplits,
487+
completedSplits,
488+
456489
cumulativeUserMemory,
457490
cumulativeTotalMemory,
458491
succinctBytes(userMemoryReservation),
@@ -534,6 +567,10 @@ public static QueryStats immediateFailureQueryStats()
534567
0,
535568
0,
536569
0,
570+
0,
571+
0,
572+
0,
573+
0,
537574
new DataSize(0, BYTE),
538575
new DataSize(0, BYTE),
539576
new DataSize(0, BYTE),
@@ -711,6 +748,30 @@ public int getCompletedDrivers()
711748
return completedDrivers;
712749
}
713750

751+
@JsonProperty
752+
public int getTotalSplits()
753+
{
754+
return totalSplits;
755+
}
756+
757+
@JsonProperty
758+
public int getQueuedSplits()
759+
{
760+
return queuedSplits;
761+
}
762+
763+
@JsonProperty
764+
public int getRunningSplits()
765+
{
766+
return runningSplits;
767+
}
768+
769+
@JsonProperty
770+
public int getCompletedSplits()
771+
{
772+
return completedSplits;
773+
}
774+
714775
@JsonProperty
715776
public double getCumulativeUserMemory()
716777
{

‎presto-main/src/main/java/com/facebook/presto/execution/StageExecutionInfo.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ public static StageExecutionInfo create(
116116
taskStatsAggregator.blockedDrivers,
117117
taskStatsAggregator.completedDrivers,
118118

119+
taskStatsAggregator.totalSplits,
120+
taskStatsAggregator.queuedSplits,
121+
taskStatsAggregator.runningSplits,
122+
taskStatsAggregator.completedSplits,
123+
119124
taskStatsAggregator.cumulativeUserMemory,
120125
taskStatsAggregator.cumulativeTotalMemory,
121126
succinctBytes(taskStatsAggregator.userMemoryReservation),
@@ -249,6 +254,11 @@ private static class TaskStatsAggregator
249254
private int blockedDrivers;
250255
private int completedDrivers;
251256

257+
private int totalSplits;
258+
private int queuedSplits;
259+
private int runningSplits;
260+
private int completedSplits;
261+
252262
private double cumulativeUserMemory;
253263
private double cumulativeTotalMemory;
254264
private long userMemoryReservation;
@@ -294,6 +304,11 @@ public void processTaskStats(TaskStats taskStats)
294304
blockedDrivers += taskStats.getBlockedDrivers();
295305
completedDrivers += taskStats.getCompletedDrivers();
296306

307+
totalSplits += taskStats.getTotalSplits();
308+
queuedSplits += taskStats.getQueuedSplits();
309+
runningSplits += taskStats.getRunningSplits();
310+
completedSplits += taskStats.getCompletedSplits();
311+
297312
cumulativeUserMemory += taskStats.getCumulativeUserMemory();
298313
cumulativeTotalMemory += taskStats.getCumulativeTotalMemory();
299314

‎presto-main/src/main/java/com/facebook/presto/execution/StageExecutionStateMachine.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,11 @@ public BasicStageExecutionStats getBasicStageStats(Supplier<Iterable<TaskInfo>>
253253
int runningDrivers = 0;
254254
int completedDrivers = 0;
255255

256+
int totalSplits = 0;
257+
int queuedSplits = 0;
258+
int runningSplits = 0;
259+
int completedSplits = 0;
260+
256261
double cumulativeUserMemory = 0;
257262
double cumulativeTotalMemory = 0;
258263
long userMemoryReservation = 0;
@@ -278,6 +283,11 @@ public BasicStageExecutionStats getBasicStageStats(Supplier<Iterable<TaskInfo>>
278283
runningDrivers += taskStats.getRunningDrivers();
279284
completedDrivers += taskStats.getCompletedDrivers();
280285

286+
totalSplits += taskStats.getTotalSplits();
287+
queuedSplits += taskStats.getQueuedSplits();
288+
runningSplits += taskStats.getRunningSplits();
289+
completedSplits += taskStats.getCompletedSplits();
290+
281291
cumulativeUserMemory += taskStats.getCumulativeUserMemory();
282292

283293
long taskUserMemory = taskStats.getUserMemoryReservationInBytes();
@@ -313,6 +323,11 @@ public BasicStageExecutionStats getBasicStageStats(Supplier<Iterable<TaskInfo>>
313323
runningDrivers,
314324
completedDrivers,
315325

326+
totalSplits,
327+
queuedSplits,
328+
runningSplits,
329+
completedSplits,
330+
316331
succinctBytes(rawInputDataSize),
317332
rawInputPositions,
318333

0 commit comments

Comments
 (0)