DC2 - Details for Job 271

Details for Job 271

Status: SUCCEEDED
Associated SQL Query: 170
Job Group: 209b98f3-f349-40a9-b4bb-8002e8da7935
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_412 { label="Stage 412 (skipped)"; subgraph cluster1923 { label="Scan parquet "; 744 [labelType="html" label="FileScanRDD [744] takeAsList at DataFrameCommonJob.java:145"]; 747 [labelType="html" label="MapPartitionsRDD [747] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1919 { label="WholeStageCodegen (2)"; 750 [labelType="html" label="MapPartitionsRDD [750] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1918 { label="Generate"; 751 [labelType="html" label="MapPartitionsRDD [751] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1913 { label="WholeStageCodegen (3)"; 752 [labelType="html" label="MapPartitionsRDD [752] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1912 { label="Generate"; 753 [labelType="html" label="MapPartitionsRDD [753] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1907 { label="WholeStageCodegen (4)"; 759 [labelType="html" label="MapPartitionsRDD [759] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1905 { label="ShuffleQueryStage"; 761 [labelType="html" label="MapPartitionsRDD [761] takeAsList at DataFrameCommonJob.java:145"]; } } 744->747; 747->750; 750->751; 751->752; 752->753; 753->759; 759->761; }

digraph G { subgraph clusterstage_413 { label="Stage 413 (skipped)"; subgraph cluster2227 { label="CustomShuffleReader"; 855 [labelType="html" label="ShuffledRowRDD [855] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2218 { label="WholeStageCodegen (9)"; 856 [labelType="html" label="MapPartitionsRDD [856] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2217 { label="ShuffleQueryStage"; 857 [labelType="html" label="MapPartitionsRDD [857] takeAsList at DataFrameCommonJob.java:145"]; } } 855->856; 856->857; }

761,855

digraph G { subgraph clusterstage_414 { label="Stage 414 (skipped)"; subgraph cluster1963 { label="Scan parquet "; 764 [labelType="html" label="FileScanRDD [764] takeAsList at DataFrameCommonJob.java:145"]; 765 [labelType="html" label="MapPartitionsRDD [765] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1953 { label="WholeStageCodegen (5)"; 770 [labelType="html" label="MapPartitionsRDD [770] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1952 { label="ShuffleQueryStage"; 771 [labelType="html" label="MapPartitionsRDD [771] takeAsList at DataFrameCommonJob.java:145"]; } } 764->765; 765->770; 770->771; }

digraph G { subgraph clusterstage_415 { label="Stage 415 (skipped)"; subgraph cluster2311 { label="CustomShuffleReader"; 877 [labelType="html" label="ShuffledRowRDD [877] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2308 { label="WholeStageCodegen (10)"; 878 [labelType="html" label="MapPartitionsRDD [878] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2315 { label="CustomShuffleReader"; 879 [labelType="html" label="ShuffledRowRDD [879] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2312 { label="WholeStageCodegen (11)"; 880 [labelType="html" label="MapPartitionsRDD [880] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2307 { label="SortMergeJoin"; 881 [labelType="html" label="ZippedPartitionsRDD2 [881] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2299 { label="WholeStageCodegen (12)"; 882 [labelType="html" label="MapPartitionsRDD [882] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2298 { label="ShuffleQueryStage"; 883 [labelType="html" label="MapPartitionsRDD [883] takeAsList at DataFrameCommonJob.java:145"]; } } 877->878; 879->880; 878->881; 880->881; 881->882; 882->883; }

857,877

771,879

digraph G { subgraph clusterstage_416 { label="Stage 416"; subgraph cluster2434 { label="CustomShuffleReader"; 901 [labelType="html" label="ShuffledRowRDD [901] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2431 { label="WholeStageCodegen (13)"; 902 [labelType="html" label="MapPartitionsRDD [902] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2435 { label="mapPartitionsInternal"; 903 [labelType="html" label="MapPartitionsRDD [903] takeAsList at DataFrameCommonJob.java:145"]; } } 901->902; 902->903; }

883,901

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
416	default	DataFrameCommonJob_dataExecOnCreation takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2025/11/07 08:59:26	26 ms	4/4			1994.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

416

default

DataFrameCommonJob_dataExecOnCreation takeAsList at DataFrameCommonJob.java:145 +details

org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927)
com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145)
com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2025/11/07 08:59:26

26 ms

4/4

1994.0 B

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
415	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/13
414	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/15
413	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/13
412	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/12

Page: