[SPARK-44153][CORE][UI] Support Heap Histogram column in Executors tab #41709
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| | ||
| package org.apache.spark.ui.exec | ||
| | ||
| import javax.servlet.http.HttpServletRequest | ||
| | ||
| import scala.xml.{Node, Text} | ||
| | ||
| import org.apache.spark.SparkContext | ||
| import org.apache.spark.ui.{SparkUITab, UIUtils, WebUIPage} | ||
| | ||
| private[ui] class ExecutorHeapHistogramPage( | ||
| parent: SparkUITab, | ||
| sc: Option[SparkContext]) extends WebUIPage("heapHistogram") { | ||
| | ||
| // Match the lines containing object informations | ||
| val pattern = """\s*([0-9]+):\s+([0-9]+)\s+([0-9]+)\s+(\S+)(.*)""".r | ||
| | ||
| def render(request: HttpServletRequest): Seq[Node] = { | ||
| val executorId = Option(request.getParameter("executorId")).map { executorId => | ||
| UIUtils.decodeURLParameter(executorId) | ||
| }.getOrElse { | ||
| throw new IllegalArgumentException(s"Missing executorId parameter") | ||
| } | ||
| val time = System.currentTimeMillis() | ||
| val maybeHeapHistogram = sc.get.getExecutorHeapHistogram(executorId) | ||
| | ||
| val content = maybeHeapHistogram.map { heapHistogram => | ||
| val rows = heapHistogram.map { row => | ||
| row match { | ||
| case pattern(rank, instances, bytes, name, module) => | ||
| <tr class="accordion-heading"> | ||
| <td>{rank}</td> | ||
| <td>{instances}</td> | ||
| <td>{bytes}</td> | ||
| <td>{name}</td> | ||
| <td>{module}</td> | ||
| </tr> | ||
| case pattern(rank, instances, bytes, name) => | ||
| <tr class="accordion-heading"> | ||
| <td>{rank}</td> | ||
| <td>{instances}</td> | ||
| <td>{bytes}</td> | ||
| <td>{name}</td> | ||
| <td></td> | ||
| </tr> | ||
| case _ => | ||
| // Ignore the first two lines and the last line | ||
| Member Author There was a problem hiding this comment. In addition, we will ignore all irregular messages in addition to these three lines. Member There was a problem hiding this comment. So the error will go to Executor log, and in UI it just doesn't show heap dump (empty)? Member Author There was a problem hiding this comment. Yes, correct. We log and return spark/core/src/main/scala/org/apache/spark/SparkContext.scala Lines 756 to 775 in 284029b | ||
| // | ||
| // num #instances #bytes class name (module) | ||
| // ------------------------------------------------------- | ||
| // ... | ||
| // Total 1267867 72845688 | ||
| } | ||
| } | ||
| <div class="row"> | ||
| <div class="col-12"> | ||
| <p>Updated at {UIUtils.formatDate(time)}</p> | ||
| <table class={UIUtils.TABLE_CLASS_STRIPED + " accordion-group" + " sortable"}> | ||
| <thead> | ||
| <th>Rank</th> | ||
| <th>Instances</th> | ||
| <th>Bytes</th> | ||
| <th>Class Name</th> | ||
| <th>Module</th> | ||
| </thead> | ||
| <tbody>{rows}</tbody> | ||
| </table> | ||
| </div> | ||
| </div> | ||
| }.getOrElse(Text("Error fetching heap histogram")) | ||
| UIUtils.headerSparkPage(request, s"Heap Histogram for Executor $executorId", content, parent) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| | @@ -2287,6 +2287,23 @@ private[spark] object Utils extends Logging with SparkClassUtils { | |
| }.map(threadInfoToThreadStackTrace) | ||
| } | ||
| | ||
| /** Return a heap dump. Used to capture dumps for the web UI */ | ||
| def getHeapHistogram(): Array[String] = { | ||
| // From Java 9+, we can use 'ProcessHandle.current().pid()' | ||
| val pid = getProcessName().split("@").head | ||
| val builder = new ProcessBuilder("jmap", "-histo:live", pid) | ||
| Member There was a problem hiding this comment. If we get error from executing it, can we get the error back?
Member Author There was a problem hiding this comment. Which error do you have in your mind? At this layer, we don't handle it like Member Author There was a problem hiding this comment. The error message will go to Member Author There was a problem hiding this comment. I took a look at spark/core/src/main/scala/org/apache/spark/util/Utils.scala Lines 1378 to 1381 in 7398e93 Member There was a problem hiding this comment. should it respect Member Author There was a problem hiding this comment. Like you pointed, there is no problem to run Member There was a problem hiding this comment. I suppose the And if only The JDK was installed by TGZ, it was unarchived to Member Author There was a problem hiding this comment. @pan3793 . Here, the contract on However, I'm interested in this case. Could you make a small reproducer with Docker image?
Contributor There was a problem hiding this comment. @dongjoon-hyun This is an issue - we should use Also, there is no compatibility gaurantees that I am aware of between different versions of jdk and jmap (for example, jdk11 jmap against jdk17 or vice versa) - if I missed any, please do let me know ! Member Author | ||
| builder.redirectErrorStream(true) | ||
| Contributor There was a problem hiding this comment. Log errors in invocation to executor logs instead of sending it to driver as response ? Member Author There was a problem hiding this comment.
| ||
| val p = builder.start() | ||
| val r = new BufferedReader(new InputStreamReader(p.getInputStream())) | ||
| Contributor There was a problem hiding this comment. nit: This reader is not closed and/or we are not doing waitFor on the process. | ||
| val rows = ArrayBuffer.empty[String] | ||
| var line = "" | ||
| while (line != null) { | ||
| if (line.nonEmpty) rows += line | ||
| line = r.readLine() | ||
| } | ||
| rows.toArray | ||
| Contributor There was a problem hiding this comment. Use Member Author There was a problem hiding this comment.
For this one, I was thinking about adding a new configuration to limit the results like Top 100 or Top 1000. I'll handle this with that new configuration together. | ||
| } | ||
| | ||
| def getThreadDumpForThread(threadId: Long): Option[ThreadStackTrace] = { | ||
| if (threadId <= 0) { | ||
| None | ||
| | ||
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Java 8 JRE doesn't have
jmap. For Java 8 JDK users, they are able this manually.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Here is the answer for
jmapquestion, @viirya .