Spark eventlog

Eventlog 示例

{
    "Event": "org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart",
    "executionId": 0,
    "rootExecutionId": 0,
    "desc ription": "select round(a, 2), a from double_table",
    "details": "org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMetho    dAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:498)\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain    (SparkSubmit.scala:1029)\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)",
    "physicalPlanDescription": "==   Physical Plan ==\n* Project (3)\n+- * ColumnarToRow (2)\n   +- Scan parquet spark_catalog.default.double_table (1)\n    \n\n(1) Scan parquet spark_catalog.default.double_table\nOutput [1]: [a#0]\nBatched: true\nLocation: InMemoryFileInde    x [file:/home/hadoop/files/double_table]\nReadSchema: struct<a:double>\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [a#0]\n\n(3) Project [codegen id : 1]\nOutput [2]: [round(a#0, 2) AS round(a, 2)#1, a#0]\nInput [1]: [a#0]\n\n",
    " sparkPlanInfo": {
        "nodeName": "WholeStageCodegen (1)",
        "simpleString": "WholeStageCodegen (1)",
        "children": [
            {
                "nodeName": "Project",
                "simpleString": "Project [round(a#0, 2) AS round(a, 2)#1, a#0]",
                "children": [
                    {
                        "nodeName": "ColumnarToRow",
                        "simple String": "ColumnarToRow",
                        "children": [
                            {
                                "nodeName": "InputAdapter",
                                "simpleString": "InputAdapter",
                                "children": [
                                    {
                                        "nodeName": "Scan parquet spark_catalog.default.double_table",
                                        "simpleString": "FileScan parquet spark_catalog.default.double_table    [a#0] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[file:/home/hadoop/files/d    ouble_table], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<a:double>",
                                        "children": [

                                        ],
                                        "metadata": {
                                            "Locat ion": "InMemoryFileIndex(1 paths)[file:/home/hadoop/files/double_table]",
                                            "ReadSchema": "struct<a:double>",
                                            "Format": "Par    quet",
                                            "Batched": "true",
                                            "PartitionFilters": "[]",
                                            "PushedFilters": "[]",
                                            "DataFilters": "[]"
                                        },
                                        "metrics": [
                                            {
                                                "name": "number of     files read",
                                                "accumulatorId": 5,
                                                "metricType": "sum"
                                            },
                                            {
                                                "name": "scan time",
                                                "accumulatorId": 4,
                                                "metricType": "timing"
                                            },
                                            {
                                                "nam e": "metadata time",
                                                "accumulatorId": 6,
                                                "metricType": "timing"
                                            },
                                            {
                                                "name": "size of files read",
                                                "accumulatorId": 7,
                                                "metricTyp e": "size"
                                            },
                                            {
                                                "name": "number of output rows",
                                                "accumulatorId": 3,
                                                "metricType": "sum"
                                            }
                                        ]
                                    }
                                ],
                                "metadata": {

                                },
                                "metrics": [

                                ]
                            }
                        ],
                        "met adata": {

                        },
                        "metrics": [
                            {
                                "name": "number of output rows",
                                "accumulatorId": 1,
                                "metricType": "sum"
                            },
                            {
                                "name": "number of input b    atches",
                                "accumulatorId": 2,
                                "metricType": "sum"
                            }
                        ]
                    }
                ],
                "metadata": {

                },
                "metrics": [

                ]
            }
        ],
        "metadata": {

        },
        "metrics": [
            {
                "name": "durat    ion",
                "accumulatorId": 0,
                "metricType": "timing"
            }
        ]
    },
    "time": 1741661558528,
    "modifiedConfigs": {

    },
    "jobTags": [

    ]
}

== Physical Plan ==
* Project (3)
+- * ColumnarToRow (2)
   +- Scan parquet spark_catalog.default.double_table (1)

对应于

==     Physical Plan ==\n* Project (3)\n+- * ColumnarToRow (2)\n   +- Scan parquet spark_catalog.default.double_table (1)\n    \n\n

(1) Scan parquet spark_catalog.default.double_table
Output [1]: [a#0]
Batched: true
Location: InMemoryFileIndex [file:/home/hadoop/files/double_table]
ReadSchema: struct<a:double>

(2) ColumnarToRow [codegen id : 1]
Input [1]: [a#0]

(3) Project [codegen id : 1]
Output [2]: [round(a#0, 2) AS round(a, 2)#1, a#0]
Input [1]: [a#0]

对应于

 "physicalPlanDescription": "==     Physical Plan ==\n* Project (3)\n+- * ColumnarToRow (2)\n   +- Scan parquet spark_catalog.default.double_table (1)\n    \n\n(1) Scan parquet spark_catalog.default.double_table\nOutput [1]: [a#0]\nBatched: true\nLocation: InMemoryFileIndex [file:/home/hadoop/files/double_table]\nReadSchema: struct<a:double>\n\n(2) ColumnarToRow [codegen id : 1]\nInput [1]: [a#0]\n\n(3) Project [codegen id : 1]\nOutput [2]: [round(a#0, 2) AS round(a, 2)#1, a#0]\nInput [1]: [a#0]\n\n",