DC2 - Details for Job 267

Details for Job 267

Status: SUCCEEDED
Associated SQL Query: 171
Job Group: 2b8b753c-c156-434b-a986-a39cc29071ef
Completed Stages: 1
Skipped Stages: 4

Enable zooming

digraph G { subgraph clusterstage_394 { label="Stage 394 (skipped)"; subgraph cluster1951 { label="Scan parquet "; 754 [labelType="html" label="FileScanRDD [754] takeAsList at DataFrameCommonJob.java:145"]; 755 [labelType="html" label="MapPartitionsRDD [755] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1947 { label="WholeStageCodegen (2)"; 756 [labelType="html" label="MapPartitionsRDD [756] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1946 { label="Generate"; 757 [labelType="html" label="MapPartitionsRDD [757] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1942 { label="WholeStageCodegen (3)"; 758 [labelType="html" label="MapPartitionsRDD [758] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1941 { label="Generate"; 760 [labelType="html" label="MapPartitionsRDD [760] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1925 { label="WholeStageCodegen (4)"; 762 [labelType="html" label="MapPartitionsRDD [762] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1924 { label="ShuffleQueryStage"; 763 [labelType="html" label="MapPartitionsRDD [763] takeAsList at DataFrameCommonJob.java:145"]; } } 754->755; 755->756; 756->757; 757->758; 758->760; 760->762; 762->763; }

digraph G { subgraph clusterstage_395 { label="Stage 395 (skipped)"; subgraph cluster2257 { label="CustomShuffleReader"; 861 [labelType="html" label="ShuffledRowRDD [861] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2248 { label="WholeStageCodegen (9)"; 862 [labelType="html" label="MapPartitionsRDD [862] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2247 { label="ShuffleQueryStage"; 863 [labelType="html" label="MapPartitionsRDD [863] takeAsList at DataFrameCommonJob.java:145"]; } } 861->862; 862->863; }

763,861

digraph G { subgraph clusterstage_396 { label="Stage 396 (skipped)"; subgraph cluster1982 { label="Scan parquet "; 766 [labelType="html" label="FileScanRDD [766] takeAsList at DataFrameCommonJob.java:145"]; 767 [labelType="html" label="MapPartitionsRDD [767] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1977 { label="WholeStageCodegen (5)"; 768 [labelType="html" label="MapPartitionsRDD [768] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster1976 { label="ShuffleQueryStage"; 769 [labelType="html" label="MapPartitionsRDD [769] takeAsList at DataFrameCommonJob.java:145"]; } } 766->767; 767->768; 768->769; }

digraph G { subgraph clusterstage_397 { label="Stage 397 (skipped)"; subgraph cluster2293 { label="CustomShuffleReader"; 870 [labelType="html" label="ShuffledRowRDD [870] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2290 { label="WholeStageCodegen (10)"; 871 [labelType="html" label="MapPartitionsRDD [871] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2297 { label="CustomShuffleReader"; 872 [labelType="html" label="ShuffledRowRDD [872] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2294 { label="WholeStageCodegen (11)"; 873 [labelType="html" label="MapPartitionsRDD [873] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2289 { label="SortMergeJoin"; 874 [labelType="html" label="ZippedPartitionsRDD2 [874] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2281 { label="WholeStageCodegen (12)"; 875 [labelType="html" label="MapPartitionsRDD [875] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2280 { label="ShuffleQueryStage"; 876 [labelType="html" label="MapPartitionsRDD [876] takeAsList at DataFrameCommonJob.java:145"]; } } 870->871; 872->873; 871->874; 873->874; 874->875; 875->876; }

863,870

769,872

digraph G { subgraph clusterstage_398 { label="Stage 398"; subgraph cluster2401 { label="CustomShuffleReader"; 898 [labelType="html" label="ShuffledRowRDD [898] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2398 { label="WholeStageCodegen (13)"; 899 [labelType="html" label="MapPartitionsRDD [899] takeAsList at DataFrameCommonJob.java:145"]; } subgraph cluster2402 { label="mapPartitionsInternal"; 900 [labelType="html" label="MapPartitionsRDD [900] takeAsList at DataFrameCommonJob.java:145"]; } } 898->899; 899->900; }

876,898

Completed Stages (1)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
398	default	DataFrameCommonJob_dataExecOnCreation takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	2025/11/07 08:59:25	18 ms	2/2			1143.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

398

default

DataFrameCommonJob_dataExecOnCreation takeAsList at DataFrameCommonJob.java:145 +details

org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927)
com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145)
com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39)
com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142)
com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47)
java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
java.base/java.util.concurrent.FutureTask.run(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
java.base/java.lang.Thread.run(Unknown Source)

2025/11/07 08:59:25

18 ms

2/2

1143.0 B

Page:

Skipped Stages (4)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total
397	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/13
396	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/15
395	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/13
394	default	takeAsList at DataFrameCommonJob.java:145 +details org.apache.spark.sql.Dataset.takeAsList(Dataset.scala:2927) com.adb.hdh.spark.job.impl.DataFrameCommonJob.take(DataFrameCommonJob.java:145) com.adb.hdh.spark.job.impl.ExecDataJob.exec(ExecDataJob.java:39) com.adb.hdh.spark.jobs.SparkJobRunnerService.runJob(SparkJobRunnerService.java:142) com.adb.hdh.spark.jobs.AbstractJobThread.run(AbstractJobThread.java:47) java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) java.base/java.util.concurrent.FutureTask.run(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) java.base/java.lang.Thread.run(Unknown Source)	Unknown	Unknown	0/12

Page: