{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4",
"toc_visible": true,
"collapsed_sections": [
"LHc1-i1dypOz",
"TLCmpDVLMtDq"
],
"authorship_tag": "ABX9TyM5I8/Mo0jXIWmhvLY7dooy",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"
]
},
{
"cell_type": "markdown",
"source": [
"\n",
"# Run Apache Spark on Google Colab\n",
"\n",
"This is a compact guide on how to set up Apache Spark on Google Colab.\n",
"\n",
"A more detailed walkthrough of how to setup Spark on a single machine in *standalone mode* is presented in in [Hadoop: Setting up Spark Standalone on Google Colab](https://github.com/groda/big_data/blob/master/Hadoop_Setting_up_Spark_Standalone_on_Google_Colab.ipynb)."
],
"metadata": {
"id": "DeF-OtwDrXVY"
}
},
{
"cell_type": "markdown",
"source": [
"## Setup Spark\n",
"\n",
"Setup Apache Spark in 1️⃣ 2️⃣ 3️⃣ 4️⃣ steps (step 0️⃣ is the Java installation, which is skipped because Java is available in Google Colab).\n",
"\n",
"The following code should also run on any Ubuntu machine or Docker container except for the Web servers links."
],
"metadata": {
"id": "LHc1-i1dypOz"
}
},
{
"cell_type": "code",
"source": [
"import requests\n",
"import subprocess\n",
"import os\n",
"import re\n",
"import socket\n",
"import shutil\n",
"import time\n",
"import sys\n",
"\n",
"def run(cmd):\n",
" # run a shell command\n",
" try:\n",
" # Run the command and capture stdout and stderr\n",
" subprocess_output = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
" # Access stdout (stderr redirected to stdout)\n",
" stdout_result = subprocess_output.stdout.strip().splitlines()[-1]\n",
" # Process the results as needed\n",
" print(f'✅ {stdout_result}')\n",
" return stdout_result\n",
" except subprocess.CalledProcessError as e:\n",
" # Handle the error if the command returns a non-zero exit code\n",
" print(f\"Command failed with return code {e.returncode}\")\n",
" print(\"stdout:\", e.stdout)\n",
"\n",
"def is_java_installed():\n",
" return shutil.which(\"java\")\n",
"\n",
"def install_java():\n",
" # Uncomment and modify the desired version\n",
" # java_version= 'openjdk-11-jre-headless'\n",
" # java_version= 'default-jre'\n",
" # java_version= 'openjdk-17-jre-headless'\n",
" # java_version= 'openjdk-18-jre-headless'\n",
" java_version= 'openjdk-19-jre-headless'\n",
" os.environ['JAVA_HOME'] = ' /usr/lib/jvm/java-19-openjdk-amd64'\n",
" print(f\"Java not found. Installing {java_version} ... (this might take a while)\")\n",
" try:\n",
" cmd = f\"apt install -y {java_version}\"\n",
" subprocess_output = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
" stdout_result = subprocess_output.stdout\n",
" # Process the results as needed\n",
" print(f'✅ Done installing Java {java_version}')\n",
" except subprocess.CalledProcessError as e:\n",
" # Handle the error if the command returns a non-zero exit code\n",
" print(f\"Command failed with return code {e.returncode}\")\n",
" print(\"stdout:\", e.stdout)\n",
"\n",
"print(\"\\n0️⃣ Install Java if not available\")\n",
"if is_java_installed():\n",
" print(\"✅ Java is already installed.\")\n",
"else:\n",
" install_java()\n",
"\n",
"print(\"\\n1️⃣ Download and install Hadoop and Spark\")\n",
"# URL for downloading Hadoop and Spark\n",
"SPARK_VERSION = \"3.5.3\"\n",
"HADOOP_SPARK_URL = \"https://dlcdn.apache.org/spark/spark-\" + SPARK_VERSION + \\\n",
" \"/spark-\" + SPARK_VERSION + \"-bin-hadoop3.tgz\"\n",
"r = requests.head(HADOOP_SPARK_URL)\n",
"if r.status_code >= 200 and r.status_code < 400:\n",
" print(f'✅ {HADOOP_SPARK_URL} was found')\n",
"else:\n",
" SPARK_CDN = \"https://dlcdn.apache.org/spark/\"\n",
" print(f'⚠️ {HADOOP_SPARK_URL} was NOT found. \\nCheck for available Spark versions in {SPARK_CDN}')\n",
"\n",
"# set some environment variables\n",
"os.environ['SPARK_HOME'] = os.path.join(os.getcwd(), os.path.splitext(os.path.basename(HADOOP_SPARK_URL))[0])\n",
"os.environ['PATH'] = ':'.join([os.path.join(os.environ['SPARK_HOME'], 'bin'), os.environ['PATH']])\n",
"os.environ['PATH'] = ':'.join([os.path.join(os.environ['SPARK_HOME'], 'sbin'), os.environ['PATH']])\n",
"\n",
"# download Spark\n",
"# using --no-clobber option will prevent wget from downloading file if already present\n",
"# shell command: wget --no-clobber $HADOOP_SPARK_URL\n",
"cmd = f\"wget --no-clobber {HADOOP_SPARK_URL}\"\n",
"run(cmd)\n",
"\n",
"# uncompress\n",
"try:\n",
" # Run the command and capture stdout and stderr\n",
" cmd = \"([ -d $(basename {0}|sed 's/\\.[^.]*$//') ] && echo -n 'Folder already exists') || (tar xzf $(basename {0}) && echo 'Uncompressed Spark distribution')\"\n",
" subprocess_output = subprocess.run(cmd.format(HADOOP_SPARK_URL), shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
" # Access stdout (stderr redirected to stdout)\n",
" stdout_result = subprocess_output.stdout\n",
" # Process the results as needed\n",
" print(f'✅ {stdout_result}')\n",
"\n",
"except subprocess.CalledProcessError as e:\n",
" # Handle the error if the command returns a non-zero exit code\n",
" print(f\"Command failed with return code {e.returncode}\")\n",
" print(\"stdout:\", e.stdout)\n",
"\n",
"\n",
"print(\"\\n2️⃣ Start Spark engine\")\n",
"# start master\n",
"# shell command: $SPARK_HOME/sbin/start-master.sh\n",
"cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-master.sh')\n",
"run(cmd)\n",
"cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-master.sh')\n",
"out = run(cmd)\n",
"\n",
"# start one worker (first stop it in case it's already running)\n",
"# shell command: $SPARK_HOME/sbin/start-worker.sh spark://${HOSTNAME}:7077\n",
"cmd = [os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-worker.sh')]\n",
"run(cmd)\n",
"cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-worker.sh') + ' ' + 'spark://'+socket.gethostname()+':7077'\n",
"run(cmd)\n",
"\n",
"print(\"\\n3️⃣ Start Master Web UI\")\n",
"# get master UI's port number\n",
"# the subprocess that's starting the master with start-master.sh\n",
"# might still not be ready with assigning the port number at this point\n",
"# therefore we check the logfile a few times (attempts=5) to see if the port\n",
"# has been assigned. This might take 1-2 seconds.\n",
"\n",
"master_log = out.partition(\"logging to\")[2].strip()\n",
"print(\"Search for port number in log file {}\".format(master_log))\n",
"attempts = 10\n",
"search_pattern = \"Successfully started service 'MasterUI' on port (\\d+)\"\n",
"found = False\n",
"for i in range(attempts):\n",
" if not found:\n",
" with open(master_log) as log:\n",
" found = re.search(search_pattern, log.read())\n",
" if found:\n",
" webUIport = found.group(1)\n",
" print(f\"✅ Master UI is available at localhost:{webUIport} (attempt nr. {i})\")\n",
" break\n",
" else:\n",
" time.sleep(2) # need to try until port information is found in the logfile\n",
" i+=1\n",
"if not found:\n",
" print(\"Could not find port for Master Web UI\\n\")\n",
"\n",
"IN_COLAB = 'google.colab' in sys.modules\n",
"if IN_COLAB:\n",
" # serve the Web UI on Colab\n",
" print(\"Click on the link below to open the Spark Web UI 🚀\")\n",
" from google.colab import output\n",
" output.serve_kernel_port_as_window(webUIport)\n",
"\n",
"print(\"\\n4️⃣ Start history server\")\n",
"# start history server\n",
"# shell command: mkdir -p /tmp/spark-events\n",
"# shell command: $SPARK_HOME/sbin/start-history-server.sh\n",
"spark_events_dir = os.path.join('/tmp', 'spark-events')\n",
"if not os.path.exists(spark_events_dir):\n",
" os.mkdir(spark_events_dir)\n",
"cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'stop-history-server.sh')\n",
"run(cmd)\n",
"cmd = os.path.join(os.environ['SPARK_HOME'], 'sbin', 'start-history-server.sh')\n",
"run(cmd)\n",
"\n",
"if IN_COLAB:\n",
" # serve the History Server\n",
" print(\"Click on the link below to open the Spark History Server Web UI 🚀\")\n",
" output.serve_kernel_port_as_window(18080)\n"
],
"metadata": {
"id": "j0ya6VMzXbKe",
"outputId": "3c4018ec-cda3-43c8-b02e-b3c95de5e083",
"colab": {
"base_uri": "https://localhost:8080/",
"height": 590
}
},
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"0️⃣ Install Java if not available\n",
"✅ Java is already installed.\n",
"\n",
"1️⃣ Download and install Hadoop and Spark\n",
"✅ https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz was found\n",
"✅ 2024-10-07 11:49:46 (54.8 MB/s) - ‘spark-3.5.3-bin-hadoop3.tgz’ saved [400864419/400864419]\n",
"✅ Uncompressed Spark distribution\n",
"\n",
"\n",
"2️⃣ Start Spark engine\n",
"✅ no org.apache.spark.deploy.master.Master to stop\n",
"✅ starting org.apache.spark.deploy.master.Master, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.master.Master-1-651dec53631c.out\n",
"✅ no org.apache.spark.deploy.worker.Worker to stop\n",
"✅ starting org.apache.spark.deploy.worker.Worker, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.worker.Worker-1-651dec53631c.out\n",
"\n",
"3️⃣ Start Master Web UI\n",
"Search for port number in log file /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.master.Master-1-651dec53631c.out\n",
"✅ Master UI is available at localhost:8081 (attempt nr. 3)\n",
"Click on the link below to open the Spark Web UI 🚀\n",
"\u001b[31mWarning: This function may stop working due to changes in browser security.\n",
"Try `serve_kernel_port_as_iframe` instead. \u001b[0m\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
""
],
"application/javascript": [
"(async (port, path, text, element) => {\n",
" if (!google.colab.kernel.accessAllowed) {\n",
" return;\n",
" }\n",
" element.appendChild(document.createTextNode(''));\n",
" const url = await google.colab.kernel.proxyPort(port);\n",
" const anchor = document.createElement('a');\n",
" anchor.href = new URL(path, url).toString();\n",
" anchor.target = '_blank';\n",
" anchor.setAttribute('data-href', url + path);\n",
" anchor.textContent = text;\n",
" element.appendChild(anchor);\n",
" })(8081, \"/\", \"https://localhost:8081/\", window.element)"
]
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"4️⃣ Start history server\n",
"✅ no org.apache.spark.deploy.history.HistoryServer to stop\n",
"✅ starting org.apache.spark.deploy.history.HistoryServer, logging to /content/spark-3.5.3-bin-hadoop3/logs/spark--org.apache.spark.deploy.history.HistoryServer-1-651dec53631c.out\n",
"Click on the link below to open the Spark History Server Web UI 🚀\n",
"\u001b[31mWarning: This function may stop working due to changes in browser security.\n",
"Try `serve_kernel_port_as_iframe` instead. \u001b[0m\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
""
],
"application/javascript": [
"(async (port, path, text, element) => {\n",
" if (!google.colab.kernel.accessAllowed) {\n",
" return;\n",
" }\n",
" element.appendChild(document.createTextNode(''));\n",
" const url = await google.colab.kernel.proxyPort(port);\n",
" const anchor = document.createElement('a');\n",
" anchor.href = new URL(path, url).toString();\n",
" anchor.target = '_blank';\n",
" anchor.setAttribute('data-href', url + path);\n",
" anchor.textContent = text;\n",
" element.appendChild(anchor);\n",
" })(18080, \"/\", \"https://localhost:18080/\", window.element)"
]
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"## Run a couple of examples\n",
"\n",
"We are going to run pre-built Java examples that come with the Spark distribution."
],
"metadata": {
"id": "MSJjuhaipr75"
}
},
{
"cell_type": "markdown",
"source": [
"### Run the Java application `SparkPi` to estimate $\\pi$\n",
"\n",
"Run the pre-built example `org.apache.spark.examples.SparkPi` that comes with the Spark distribution using `bash` to submit the job.\n",
"\n",
"$100$ is the number of iterations."
],
"metadata": {
"id": "TLCmpDVLMtDq"
}
},
{
"cell_type": "code",
"source": [
"%%bash\n",
"\n",
"EXAMPLES_JAR=$(find $SPARK_HOME/examples/jars/ -name \"spark-examples*\")\n",
"\n",
"$SPARK_HOME/bin/spark-submit \\\n",
" --class org.apache.spark.examples.SparkPi \\\n",
" --master spark://${HOSTNAME}:7077 \\\n",
" --conf spark.eventLog.enabled=true \\\n",
" $EXAMPLES_JAR \\\n",
" 100 \\\n",
" 2>/tmp/SparkPi_bash.log"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "yut-LNV-MuZX",
"outputId": "7c622618-1229-4653-e2da-da3344134c87"
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Pi is roughly 3.1415227141522712\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"### Java Word Count\n",
"\n",
"The source code for this example is available on GitHub at: https://github.com/apache/spark/blob/master/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java"
],
"metadata": {
"id": "NrwGkW_OM43U"
}
},
{
"cell_type": "code",
"source": [
"%%bash\n",
"\n",
"echo \"Downloading file ...\"\n",
"URL=\"https://www.gutenberg.org/cache/epub/71036/pg71036.txt\"\n",
"([ -f datafile.txt ] && echo \"File already exists\") || curl -o datafile.txt $URL"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "aKHJYsGPMqFb",
"outputId": "f076699a-22b2-496a-dde6-79a2dca13a5e"
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Downloading file ...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 47566 100 47566 0 0 244k 0 --:--:-- --:--:-- --:--:-- 245k\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"%%bash\n",
"\n",
"EXAMPLES_JAR=$(find $SPARK_HOME/examples/jars/ -name \"spark-examples*\")\n",
"\n",
"$SPARK_HOME/bin/spark-submit \\\n",
" --class org.apache.spark.examples.JavaWordCount \\\n",
" --master spark://${HOSTNAME}:7077 \\\n",
" --conf spark.eventLog.enabled=true \\\n",
" $EXAMPLES_JAR \\\n",
" datafile.txt \\\n",
" 2>/tmp/JavaWordCount.log \\\n",
" 1>/tmp/JavaWordCount.out\n",
"\n",
"head /tmp/JavaWordCount.out"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "oSKF08RRm43X",
"outputId": "4ff06d4e-4d64-44ce-de09-171721db7b84"
},
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"young: 5\n",
"Unless: 1\n",
"seriousness,: 1\n",
"errors,: 1\n",
"repent.: 1\n",
"(or: 3\n",
"Printers.: 1\n",
"full: 10\n",
"previous: 1\n",
"secure: 1\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"To download a larger amount of data from the Gutenberg collection, see the tutorial [Explore and download books from the Gutenberg Books collection](https://github.com/groda/big_data/blob/master/GutenbergBooks.ipynb) from the BDb repository."
],
"metadata": {
"id": "L4xFcb_m42Ex"
}
},
{
"cell_type": "markdown",
"source": [
"## Where to find things"
],
"metadata": {
"id": "caA8NuINpmup"
}
},
{
"cell_type": "markdown",
"source": [
"### Spark Web UI\n",
"\n",
"The Spark Web UI is available at:"
],
"metadata": {
"id": "-yR4urmDp731"
}
},
{
"cell_type": "code",
"source": [
"if IN_COLAB:\n",
" from google.colab.output import eval_js\n",
" print(eval_js( \"google.colab.kernel.proxyPort(\" + str(webUIport) + \")\" ))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "rbzRLwxwzBfu",
"outputId": "70228f3d-2dfc-43e9-fe0b-24794a70503f"
},
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"https://7ct245v95zw-496ff2e9c6d22116-8081-colab.googleusercontent.com/\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"In the free tier of Google Colab this functionality might not be available (see https://research.google.com/colaboratory/faq.html#limitations-and-restrictions). As an alternative, you can use [ngrok](https://ngrok.com/) after signing up for a free account."
],
"metadata": {
"id": "qk8RkB-PZ5G_"
}
},
{
"cell_type": "markdown",
"source": [
"#### Use ngrok to access the Web UI"
],
"metadata": {
"id": "fsXfeH_GZs1I"
}
},
{
"cell_type": "markdown",
"source": [
"Check the NGROK box below if you want to use ngrok (by default this is set to `False`)."
],
"metadata": {
"id": "2JD-qpP5Zybf"
}
},
{
"cell_type": "code",
"source": [
"# you should set this to True\n",
"NGROK = False #@param {type:\"boolean\"}"
],
"metadata": {
"id": "BZd46bH6ZyRo"
},
"execution_count": 6,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"We are going to use the Python ngrok client `pyngrok` (see the [Colab example](https://pyngrok.readthedocs.io/en/latest/integrations.html#colab-http-example))."
],
"metadata": {
"id": "iBcnTWPPaMqw"
}
},
{
"cell_type": "code",
"source": [
"if NGROK:\n",
" !pip install pyngrok\n",
" from pyngrok import ngrok, conf\n",
" import getpass\n",
"\n",
" print(\"Enter your authtoken, which can be copied from https://dashboard.ngrok.com/get-started/your-authtoken\")\n",
" authtoken = getpass.getpass()\n",
" conf.get_default().auth_token = authtoken"
],
"metadata": {
"id": "KcpLdqcnaS7q"
},
"execution_count": 7,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"**Note:** It might be necessary to close other open sessions by stopping ngrok agents in [https://dashboard.ngrok.com/tunnels/agents](https://dashboard.ngrok.com/tunnels/agents) (the ngrok free tier has a limit of 1 simultaneous ngrok agent sessions)."
],
"metadata": {
"id": "vCbeJO6Dadej"
}
},
{
"cell_type": "markdown",
"source": [
"After entering the ngrok authorization token, you can open a connection."
],
"metadata": {
"id": "Hz2JOHqrbDAC"
}
},
{
"cell_type": "code",
"source": [
"if NGROK:\n",
" # close all existing connections (https://pyngrok.readthedocs.io/en/latest/#get-active-tunnels)\n",
" tunnels = ngrok.get_tunnels()\n",
" if tunnels:\n",
" map(lambda t: ngrok.disconnect(t.public_url), tunnels)\n",
" # Open a ngrok tunnel to the HTTP server\n",
" public_url = ngrok.connect(webUIport).public_url\n",
" print(f'Click on {public_url} to open the Spark Master Web UI')"
],
"metadata": {
"id": "1AXh71YPbF3F"
},
"execution_count": 8,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"You can safely ignore the warning since we are not disclosing any confidential information and proceed with clicking on the \"Visit site\" button."
],
"metadata": {
"id": "JJ3XT-_2aqrl"
}
},
{
"cell_type": "markdown",
"source": [
""
],
"metadata": {
"id": "V26SqdgrFXXF"
}
},
{
"cell_type": "markdown",
"source": [
""
],
"metadata": {
"id": "kl1fKUyFvzud"
}
},
{
"cell_type": "markdown",
"source": [
"### Spark History Server\n",
"\n",
"The Spark History Server is available at:"
],
"metadata": {
"id": "t-ZCGKbuqwqn"
}
},
{
"cell_type": "code",
"source": [
"if IN_COLAB:\n",
" from google.colab.output import eval_js\n",
" print(eval_js( \"google.colab.kernel.proxyPort(\" + str(18080) + \")\" ))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 35
},
"id": "vmx3UOgwqM91",
"outputId": "72d56012-2e8f-46ce-89f5-3ad22b90038c"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"https://5tsehcxns9p-496ff2e9c6d22116-18080-colab.googleusercontent.com/\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"With ngrok:"
],
"metadata": {
"id": "DnvirdlKbY64"
}
},
{
"cell_type": "code",
"source": [
"if NGROK:\n",
" # Open a ngrok tunnel to the HTTP server\n",
" public_url = ngrok.connect(18080).public_url\n",
" print(f'Click on {public_url} to open the Spark Master Web UI')"
],
"metadata": {
"id": "ev-pGUHIbbAo"
},
"execution_count": 10,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
""
],
"metadata": {
"id": "SmTVGSACv3qT"
}
},
{
"cell_type": "markdown",
"source": [
"### Logs for the Spark Master"
],
"metadata": {
"id": "52qJ8kVoridr"
}
},
{
"cell_type": "code",
"source": [
"!head -20 $SPARK_HOME/logs/*Master*.out"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "CcnrxS8drsaW",
"outputId": "5527225a-6ace-45d2-d0e1-ff6820ce1f9e"
},
"execution_count": 11,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Spark Command: /usr/lib/jvm/java-11-openjdk-amd64/bin/java -cp /content/spark-3.5.3-bin-hadoop3/conf/:/content/spark-3.5.3-bin-hadoop3/jars/* -Xmx1g org.apache.spark.deploy.master.Master --host 651dec53631c --port 7077 --webui-port 8080\n",
"========================================\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
"24/10/07 11:50:02 INFO Master: Started daemon with process name: 732@651dec53631c\n",
"24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for TERM\n",
"24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for HUP\n",
"24/10/07 11:50:02 INFO SignalUtils: Registering signal handler for INT\n",
"24/10/07 11:50:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
"24/10/07 11:50:04 INFO SecurityManager: Changing view acls to: root\n",
"24/10/07 11:50:04 INFO SecurityManager: Changing modify acls to: root\n",
"24/10/07 11:50:04 INFO SecurityManager: Changing view acls groups to: \n",
"24/10/07 11:50:04 INFO SecurityManager: Changing modify acls groups to: \n",
"24/10/07 11:50:04 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: root; groups with view permissions: EMPTY; users with modify permissions: root; groups with modify permissions: EMPTY\n",
"24/10/07 11:50:06 INFO Utils: Successfully started service 'sparkMaster' on port 7077.\n",
"24/10/07 11:50:06 INFO Master: Starting Spark master at spark://651dec53631c:7077\n",
"24/10/07 11:50:06 INFO Master: Running Spark version 3.5.3\n",
"24/10/07 11:50:07 INFO JettyUtils: Start Jetty 0.0.0.0:8080 for MasterUI\n",
"24/10/07 11:50:07 WARN Utils: Service 'MasterUI' could not bind on port 8080. Attempting port 8081.\n",
"24/10/07 11:50:07 INFO Utils: Successfully started service 'MasterUI' on port 8081.\n",
"24/10/07 11:50:08 INFO MasterWebUI: Bound MasterWebUI to 0.0.0.0, and started at http://651dec53631c:8081\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"### Logs for the Spark Worker"
],
"metadata": {
"id": "Iy1UsWdMrskL"
}
},
{
"cell_type": "code",
"source": [
"!head -20 $SPARK_HOME/logs/*Worker*.out"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "MHoYLbCfwppf",
"outputId": "0370c301-bb71-4ee8-f954-4397bef868c5"
},
"execution_count": 12,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Spark Command: /usr/lib/jvm/java-11-openjdk-amd64/bin/java -cp /content/spark-3.5.3-bin-hadoop3/conf/:/content/spark-3.5.3-bin-hadoop3/jars/* -Xmx1g org.apache.spark.deploy.worker.Worker --webui-port 8081 spark://651dec53631c:7077\n",
"========================================\n",
"Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties\n",
"24/10/07 11:50:08 INFO Worker: Started daemon with process name: 807@651dec53631c\n",
"24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for TERM\n",
"24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for HUP\n",
"24/10/07 11:50:08 INFO SignalUtils: Registering signal handler for INT\n",
"24/10/07 11:50:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
"24/10/07 11:50:09 INFO SecurityManager: Changing view acls to: root\n",
"24/10/07 11:50:09 INFO SecurityManager: Changing modify acls to: root\n",
"24/10/07 11:50:09 INFO SecurityManager: Changing view acls groups to: \n",
"24/10/07 11:50:09 INFO SecurityManager: Changing modify acls groups to: \n",
"24/10/07 11:50:09 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: root; groups with view permissions: EMPTY; users with modify permissions: root; groups with modify permissions: EMPTY\n",
"24/10/07 11:50:10 INFO Utils: Successfully started service 'sparkWorker' on port 41689.\n",
"24/10/07 11:50:10 INFO Worker: Worker decommissioning not enabled.\n",
"24/10/07 11:50:11 INFO Worker: Starting Spark worker 172.28.0.12:41689 with 2 cores, 11.7 GiB RAM\n",
"24/10/07 11:50:11 INFO Worker: Running Spark version 3.5.3\n",
"24/10/07 11:50:11 INFO Worker: Spark home: /content/spark-3.5.3-bin-hadoop3\n",
"24/10/07 11:50:11 INFO ResourceUtils: ==============================================================\n",
"24/10/07 11:50:11 INFO ResourceUtils: No custom resources configured for spark.worker.\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"### Spark events (used by History Server)"
],
"metadata": {
"id": "7hxeai2vsPM0"
}
},
{
"cell_type": "code",
"source": [
"!head -20 /tmp/spark-events/*"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "KlqvptLesVNY",
"outputId": "615f5fb1-5cfb-45d4-b9cf-f72e49f736cb"
},
"execution_count": 13,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"==> /tmp/spark-events/app-20241007115024-0000 <==\n",
"{\"Event\":\"SparkListenerLogStart\",\"Spark Version\":\"3.5.3\"}\n",
"{\"Event\":\"SparkListenerResourceProfileAdded\",\"Resource Profile Id\":0,\"Executor Resource Requests\":{\"memory\":{\"Resource Name\":\"memory\",\"Amount\":1024,\"Discovery Script\":\"\",\"Vendor\":\"\"},\"offHeap\":{\"Resource Name\":\"offHeap\",\"Amount\":0,\"Discovery Script\":\"\",\"Vendor\":\"\"}},\"Task Resource Requests\":{\"cpus\":{\"Resource Name\":\"cpus\",\"Amount\":1.0}}}\n",
"{\"Event\":\"SparkListenerBlockManagerAdded\",\"Block Manager ID\":{\"Executor ID\":\"driver\",\"Host\":\"651dec53631c\",\"Port\":33019},\"Maximum Memory\":455501414,\"Timestamp\":1728301824810,\"Maximum Onheap Memory\":455501414,\"Maximum Offheap Memory\":0}\n",
"{\"Event\":\"SparkListenerEnvironmentUpdate\",\"JVM Information\":{\"Java Home\":\"/usr/lib/jvm/java-11-openjdk-amd64\",\"Java Version\":\"11.0.24 (Ubuntu)\",\"Scala Version\":\"version 2.12.18\"},\"Spark Properties\":{\"spark.executor.extraJavaOptions\":\"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false\",\"spark.driver.host\":\"651dec53631c\",\"spark.eventLog.enabled\":\"true\",\"spark.driver.port\":\"37939\",\"spark.jars\":\"file:/content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar\",\"spark.app.name\":\"Spark Pi\",\"spark.scheduler.mode\":\"FIFO\",\"spark.submit.pyFiles\":\"\",\"spark.app.submitTime\":\"1728301820515\",\"spark.app.startTime\":\"1728301820670\",\"spark.executor.id\":\"driver\",\"spark.driver.extraJavaOptions\":\"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false\",\"spark.app.initial.jar.urls\":\"spark://651dec53631c:37939/jars/spark-examples_2.12-3.5.3.jar\",\"spark.submit.deployMode\":\"client\",\"spark.master\":\"spark://651dec53631c:7077\",\"spark.app.id\":\"app-20241007115024-0000\"},\"Hadoop Properties\":{\"hadoop.service.shutdown.timeout\":\"30s\",\"yarn.resourcemanager.amlauncher.thread-count\":\"50\",\"yarn.sharedcache.enabled\":\"false\",\"fs.s3a.connection.maximum\":\"96\",\"yarn.nodemanager.numa-awareness.numactl.cmd\":\"/usr/bin/numactl\",\"fs.viewfs.overload.scheme.target.o3fs.impl\":\"org.apache.hadoop.fs.ozone.OzoneFileSystem\",\"fs.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3AFileSystem\",\"yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms\":\"1000\",\"yarn.timeline-service.timeline-client.number-of-async-entities-to-merge\":\"10\",\"hadoop.security.kms.client.timeout\":\"60\",\"hadoop.http.authentication.kerberos.principal\":\"HTTP/_HOST@LOCALHOST\",\"mapreduce.jobhistory.loadedjob.tasks.max\":\"-1\",\"yarn.resourcemanager.application-tag-based-placement.enable\":\"false\",\"mapreduce.framework.name\":\"local\",\"yarn.sharedcache.uploader.server.thread-count\":\"50\",\"yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min\":\"3600\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern\":\"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$\",\"tfile.fs.output.buffer.size\":\"262144\",\"yarn.app.mapreduce.am.job.task.listener.thread-count\":\"30\",\"yarn.nodemanager.node-attributes.resync-interval-ms\":\"120000\",\"yarn.nodemanager.container-log-monitor.interval-ms\":\"60000\",\"hadoop.security.groups.cache.background.reload.threads\":\"3\",\"yarn.resourcemanager.webapp.cross-origin.enabled\":\"false\",\"fs.AbstractFileSystem.ftp.impl\":\"org.apache.hadoop.fs.ftp.FtpFs\",\"fs.viewfs.overload.scheme.target.gs.impl\":\"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS\",\"hadoop.registry.secure\":\"false\",\"hadoop.shell.safely.delete.limit.num.files\":\"100\",\"mapreduce.job.acl-view-job\":\" \",\"fs.s3a.s3guard.ddb.background.sleep\":\"25ms\",\"fs.s3a.retry.limit\":\"7\",\"mapreduce.jobhistory.loadedjobs.cache.size\":\"5\",\"fs.s3a.s3guard.ddb.table.create\":\"false\",\"fs.viewfs.overload.scheme.target.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3AFileSystem\",\"yarn.nodemanager.amrmproxy.enabled\":\"false\",\"yarn.timeline-service.entity-group-fs-store.with-user-dir\":\"false\",\"mapreduce.shuffle.pathcache.expire-after-access-minutes\":\"5\",\"mapreduce.input.fileinputformat.split.minsize\":\"0\",\"yarn.resourcemanager.container.liveness-monitor.interval-ms\":\"600000\",\"yarn.resourcemanager.client.thread-count\":\"50\",\"io.seqfile.compress.blocksize\":\"1000000\",\"yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes\":\"runc\",\"fs.viewfs.overload.scheme.target.http.impl\":\"org.apache.hadoop.fs.http.HttpFileSystem\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor\":\"1.0\",\"yarn.sharedcache.checksum.algo.impl\":\"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl\",\"yarn.nodemanager.amrmproxy.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor\",\"yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size\":\"10485760\",\"mapreduce.reduce.shuffle.fetch.retry.interval-ms\":\"1000\",\"mapreduce.task.profile.maps\":\"0-2\",\"yarn.scheduler.include-port-in-node-name\":\"false\",\"yarn.nodemanager.admin-env\":\"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX\",\"yarn.resourcemanager.node-removal-untracked.timeout-ms\":\"60000\",\"mapreduce.am.max-attempts\":\"2\",\"hadoop.security.kms.client.failover.sleep.base.millis\":\"100\",\"mapreduce.jobhistory.webapp.https.address\":\"0.0.0.0:19890\",\"yarn.node-labels.fs-store.impl.class\":\"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore\",\"yarn.nodemanager.collector-service.address\":\"${yarn.nodemanager.hostname}:8048\",\"fs.trash.checkpoint.interval\":\"0\",\"mapreduce.job.map.output.collector.class\":\"org.apache.hadoop.mapred.MapTask$MapOutputBuffer\",\"yarn.resourcemanager.node-ip-cache.expiry-interval-secs\":\"-1\",\"hadoop.http.authentication.signature.secret.file\":\"*********(redacted)\",\"hadoop.jetty.logs.serve.aliases\":\"true\",\"yarn.resourcemanager.placement-constraints.handler\":\"disabled\",\"yarn.timeline-service.handler-thread-count\":\"10\",\"yarn.resourcemanager.max-completed-applications\":\"1000\",\"yarn.nodemanager.aux-services.manifest.enabled\":\"false\",\"yarn.resourcemanager.system-metrics-publisher.enabled\":\"false\",\"yarn.resourcemanager.placement-constraints.algorithm.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm\",\"yarn.sharedcache.webapp.address\":\"0.0.0.0:8788\",\"fs.s3a.select.input.csv.quote.escape.character\":\"\\\\\\\\\",\"yarn.resourcemanager.delegation.token.renew-interval\":\"*********(redacted)\",\"yarn.sharedcache.nm.uploader.replication.factor\":\"10\",\"hadoop.security.groups.negative-cache.secs\":\"30\",\"yarn.app.mapreduce.task.container.log.backups\":\"0\",\"mapreduce.reduce.skip.proc-count.auto-incr\":\"true\",\"fs.viewfs.overload.scheme.target.swift.impl\":\"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem\",\"hadoop.security.group.mapping.ldap.posix.attr.gid.name\":\"gidNumber\",\"ipc.client.fallback-to-simple-auth-allowed\":\"false\",\"yarn.nodemanager.resource.memory.enforced\":\"true\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch\":\"false\",\"yarn.client.failover-proxy-provider\":\"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider\",\"yarn.timeline-service.http-authentication.simple.anonymous.allowed\":\"true\",\"ha.health-monitor.check-interval.ms\":\"1000\",\"yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed\":\"false\",\"hadoop.metrics.jvm.use-thread-mxbean\":\"false\",\"ipc.[port_number].faircallqueue.multiplexer.weights\":\"8,4,2,1\",\"yarn.acl.reservation-enable\":\"false\",\"yarn.resourcemanager.store.class\":\"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore\",\"yarn.app.mapreduce.am.hard-kill-timeout-ms\":\"10000\",\"fs.s3a.etag.checksum.enabled\":\"false\",\"yarn.nodemanager.container-metrics.enable\":\"true\",\"ha.health-monitor.rpc.connect.max.retries\":\"1\",\"yarn.timeline-service.client.fd-clean-interval-secs\":\"60\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable\":\"false\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-ms\":\"1000\",\"hadoop.common.configuration.version\":\"3.0.0\",\"fs.s3a.s3guard.ddb.table.capacity.read\":\"0\",\"yarn.nodemanager.remote-app-log-dir-suffix\":\"logs\",\"yarn.nodemanager.container-log-monitor.dir-size-limit-bytes\":\"1000000000\",\"yarn.nodemanager.windows-container.cpu-limit.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed\":\"false\",\"file.blocksize\":\"67108864\",\"hadoop.http.idle_timeout.ms\":\"60000\",\"hadoop.registry.zk.retry.ceiling.ms\":\"60000\",\"yarn.scheduler.configuration.leveldb-store.path\":\"${hadoop.tmp.dir}/yarn/system/confstore\",\"yarn.sharedcache.store.in-memory.initial-delay-mins\":\"10\",\"mapreduce.jobhistory.principal\":\"jhs/_HOST@REALM.TLD\",\"mapreduce.map.skip.proc-count.auto-incr\":\"true\",\"fs.s3a.committer.name\":\"file\",\"mapreduce.task.profile.reduces\":\"0-2\",\"hadoop.zk.num-retries\":\"1000\",\"yarn.webapp.xfs-filter.enabled\":\"true\",\"fs.viewfs.overload.scheme.target.hdfs.impl\":\"org.apache.hadoop.hdfs.DistributedFileSystem\",\"seq.io.sort.mb\":\"100\",\"yarn.scheduler.configuration.max.version\":\"100\",\"yarn.timeline-service.webapp.https.address\":\"${yarn.timeline-service.hostname}:8190\",\"yarn.resourcemanager.scheduler.address\":\"${yarn.resourcemanager.hostname}:8030\",\"yarn.node-labels.enabled\":\"false\",\"yarn.resourcemanager.webapp.ui-actions.enabled\":\"true\",\"mapreduce.task.timeout\":\"600000\",\"yarn.sharedcache.client-server.thread-count\":\"50\",\"hadoop.security.groups.shell.command.timeout\":\"0s\",\"hadoop.security.crypto.cipher.suite\":\"AES/CTR/NoPadding\",\"yarn.nodemanager.elastic-memory-control.oom-handler\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler\",\"yarn.resourcemanager.connect.max-wait.ms\":\"900000\",\"fs.defaultFS\":\"file:///\",\"yarn.minicluster.use-rpc\":\"false\",\"ipc.[port_number].decay-scheduler.decay-factor\":\"0.5\",\"fs.har.impl.disable.cache\":\"true\",\"yarn.webapp.ui2.enable\":\"false\",\"io.compression.codec.bzip2.library\":\"system-native\",\"yarn.webapp.filter-invalid-xml-chars\":\"false\",\"yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs\":\"600\",\"fs.s3a.select.input.csv.record.delimiter\":\"\\\\n\",\"fs.s3a.change.detection.source\":\"etag\",\"ipc.[port_number].backoff.enable\":\"false\",\"yarn.nodemanager.distributed-scheduling.enabled\":\"false\",\"mapreduce.shuffle.connection-keep-alive.timeout\":\"5\",\"yarn.resourcemanager.webapp.https.address\":\"${yarn.resourcemanager.hostname}:8090\",\"yarn.webapp.enable-rest-app-submissions\":\"true\",\"mapreduce.jobhistory.address\":\"0.0.0.0:10020\",\"yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"yarn.is.minicluster\":\"false\",\"yarn.nodemanager.address\":\"${yarn.nodemanager.hostname}:0\",\"fs.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem\",\"fs.AbstractFileSystem.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3A\",\"mapreduce.task.combine.progress.records\":\"10000\",\"yarn.resourcemanager.epoch.range\":\"0\",\"yarn.resourcemanager.am.max-attempts\":\"2\",\"yarn.nodemanager.runtime.linux.runc.image-toplevel-dir\":\"/runc-root\",\"yarn.nodemanager.linux-container-executor.cgroups.hierarchy\":\"/hadoop-yarn\",\"fs.AbstractFileSystem.wasbs.impl\":\"org.apache.hadoop.fs.azure.Wasbs\",\"yarn.timeline-service.entity-group-fs-store.cache-store-class\":\"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore\",\"yarn.nodemanager.runtime.linux.runc.allowed-container-networks\":\"host,none,bridge\",\"fs.ftp.transfer.mode\":\"BLOCK_TRANSFER_MODE\",\"ipc.server.log.slow.rpc\":\"false\",\"ipc.server.reuseaddr\":\"true\",\"fs.ftp.timeout\":\"0\",\"yarn.resourcemanager.node-labels.provider.fetch-interval-ms\":\"1800000\",\"yarn.router.webapp.https.address\":\"0.0.0.0:8091\",\"yarn.nodemanager.webapp.cross-origin.enabled\":\"false\",\"fs.wasb.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem\",\"yarn.resourcemanager.auto-update.containers\":\"false\",\"yarn.app.mapreduce.am.job.committer.cancel-timeout\":\"60000\",\"yarn.scheduler.configuration.zk-store.parent-path\":\"/confstore\",\"yarn.nodemanager.default-container-executor.log-dirs.permissions\":\"710\",\"yarn.app.attempt.diagnostics.limit.kc\":\"64\",\"fs.viewfs.overload.scheme.target.swebhdfs.impl\":\"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem\",\"yarn.client.failover-no-ha-proxy-provider\":\"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider\",\"fs.s3a.change.detection.mode\":\"server\",\"ftp.bytes-per-checksum\":\"512\",\"yarn.nodemanager.resource.memory-mb\":\"-1\",\"fs.AbstractFileSystem.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.Abfs\",\"yarn.timeline-service.writer.flush-interval-seconds\":\"60\",\"fs.s3a.fast.upload.active.blocks\":\"4\",\"yarn.resourcemanager.submission-preprocessor.enabled\":\"false\",\"hadoop.security.credential.clear-text-fallback\":\"true\",\"yarn.nodemanager.collector-service.thread-count\":\"5\",\"ipc.[port_number].scheduler.impl\":\"org.apache.hadoop.ipc.DefaultRpcScheduler\",\"fs.azure.secure.mode\":\"false\",\"mapreduce.jobhistory.joblist.cache.size\":\"20000\",\"fs.ftp.host\":\"0.0.0.0\",\"yarn.timeline-service.writer.async.queue.capacity\":\"100\",\"yarn.resourcemanager.fs.state-store.num-retries\":\"0\",\"yarn.resourcemanager.nodemanager-connect-retries\":\"10\",\"yarn.nodemanager.log-aggregation.num-log-files-per-app\":\"30\",\"hadoop.security.kms.client.encrypted.key.cache.low-watermark\":\"0.3f\",\"fs.s3a.committer.magic.enabled\":\"true\",\"yarn.timeline-service.client.max-retries\":\"30\",\"dfs.ha.fencing.ssh.connect-timeout\":\"30000\",\"yarn.log-aggregation-enable\":\"false\",\"yarn.system-metrics-publisher.enabled\":\"false\",\"mapreduce.reduce.markreset.buffer.percent\":\"0.0\",\"fs.AbstractFileSystem.viewfs.impl\":\"org.apache.hadoop.fs.viewfs.ViewFs\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor\":\"1.0\",\"mapreduce.task.io.sort.factor\":\"10\",\"yarn.nodemanager.amrmproxy.client.thread-count\":\"25\",\"ha.failover-controller.new-active.rpc-timeout.ms\":\"60000\",\"yarn.nodemanager.container-localizer.java.opts\":\"-Xmx256m\",\"mapreduce.jobhistory.datestring.cache.size\":\"200000\",\"mapreduce.job.acl-modify-job\":\" \",\"yarn.nodemanager.windows-container.memory-limit.enabled\":\"false\",\"yarn.timeline-service.webapp.address\":\"${yarn.timeline-service.hostname}:8188\",\"yarn.app.mapreduce.am.job.committer.commit-window\":\"10000\",\"yarn.nodemanager.container-manager.thread-count\":\"20\",\"yarn.minicluster.fixed.ports\":\"false\",\"hadoop.tags.system\":\"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL\",\"yarn.cluster.max-application-priority\":\"0\",\"yarn.timeline-service.ttl-enable\":\"true\",\"mapreduce.jobhistory.recovery.store.fs.uri\":\"${hadoop.tmp.dir}/mapred/history/recoverystore\",\"hadoop.caller.context.signature.max.size\":\"40\",\"ipc.[port_number].decay-scheduler.backoff.responsetime.enable\":\"false\",\"yarn.client.load.resource-types.from-server\":\"false\",\"ha.zookeeper.session-timeout.ms\":\"10000\",\"ipc.[port_number].decay-scheduler.metrics.top.user.count\":\"10\",\"tfile.io.chunk.size\":\"1048576\",\"fs.s3a.s3guard.ddb.table.capacity.write\":\"0\",\"yarn.dispatcher.print-events-info.threshold\":\"5000\",\"mapreduce.job.speculative.slowtaskthreshold\":\"1.0\",\"io.serializations\":\"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization\",\"hadoop.security.kms.client.failover.sleep.max.millis\":\"2000\",\"hadoop.security.group.mapping.ldap.directory.search.timeout\":\"10000\",\"yarn.scheduler.configuration.store.max-logs\":\"1000\",\"yarn.nodemanager.node-attributes.provider.fetch-interval-ms\":\"600000\",\"fs.swift.impl\":\"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem\",\"yarn.nodemanager.local-cache.max-files-per-directory\":\"8192\",\"hadoop.http.cross-origin.enabled\":\"false\",\"hadoop.zk.acl\":\"world:anyone:rwcda\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache\":\"10\",\"mapreduce.map.sort.spill.percent\":\"0.80\",\"yarn.timeline-service.entity-group-fs-store.scan-interval-seconds\":\"60\",\"yarn.node-attribute.fs-store.impl.class\":\"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore\",\"fs.s3a.retry.interval\":\"500ms\",\"yarn.timeline-service.client.best-effort\":\"false\",\"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\":\"*********(redacted)\",\"hadoop.security.group.mapping.ldap.posix.attr.uid.name\":\"uidNumber\",\"fs.AbstractFileSystem.swebhdfs.impl\":\"org.apache.hadoop.fs.SWebHdfs\",\"yarn.nodemanager.elastic-memory-control.timeout-sec\":\"5\",\"fs.s3a.select.enabled\":\"true\",\"mapreduce.ifile.readahead\":\"true\",\"yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms\":\"300000\",\"yarn.timeline-service.reader.webapp.address\":\"${yarn.timeline-service.webapp.address}\",\"yarn.resourcemanager.placement-constraints.algorithm.pool-size\":\"1\",\"yarn.timeline-service.hbase.coprocessor.jar.hdfs.location\":\"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar\",\"hadoop.security.kms.client.encrypted.key.cache.num.refill.threads\":\"2\",\"yarn.resourcemanager.scheduler.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler\",\"yarn.app.mapreduce.am.command-opts\":\"-Xmx1024m\",\"fs.s3a.metadatastore.fail.on.write.error\":\"true\",\"hadoop.http.sni.host.check.enabled\":\"false\",\"mapreduce.cluster.local.dir\":\"${hadoop.tmp.dir}/mapred/local\",\"io.mapfile.bloom.error.rate\":\"0.005\",\"fs.client.resolve.topology.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.allowed-runtimes\":\"default\",\"yarn.sharedcache.store.class\":\"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore\",\"ha.failover-controller.graceful-fence.rpc-timeout.ms\":\"5000\",\"ftp.replication\":\"3\",\"fs.getspaceused.jitterMillis\":\"60000\",\"hadoop.security.uid.cache.secs\":\"14400\",\"mapreduce.job.maxtaskfailures.per.tracker\":\"3\",\"fs.s3a.metadatastore.impl\":\"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore\",\"io.skip.checksum.errors\":\"false\",\"yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts\":\"3\",\"yarn.timeline-service.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"fs.s3a.connection.timeout\":\"200000\",\"yarn.app.mapreduce.am.webapp.https.enabled\":\"false\",\"mapreduce.job.max.split.locations\":\"15\",\"yarn.resourcemanager.nm-container-queuing.max-queue-length\":\"15\",\"yarn.resourcemanager.delegation-token.always-cancel\":\"*********(redacted)\",\"hadoop.registry.zk.session.timeout.ms\":\"60000\",\"yarn.federation.cache-ttl.secs\":\"300\",\"mapreduce.jvm.system-properties-to-log\":\"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name\",\"yarn.resourcemanager.opportunistic-container-allocation.nodes-used\":\"10\",\"yarn.timeline-service.entity-group-fs-store.active-dir\":\"/tmp/entity-file-history/active\",\"mapreduce.shuffle.transfer.buffer.size\":\"131072\",\"yarn.timeline-service.client.retry-interval-ms\":\"1000\",\"yarn.timeline-service.flowname.max-size\":\"0\",\"yarn.http.policy\":\"HTTP_ONLY\",\"fs.s3a.socket.send.buffer\":\"8192\",\"fs.AbstractFileSystem.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.Abfss\",\"yarn.sharedcache.uploader.server.address\":\"0.0.0.0:8046\",\"yarn.resourcemanager.delegation-token.max-conf-size-bytes\":\"*********(redacted)\",\"hadoop.http.authentication.token.validity\":\"*********(redacted)\",\"mapreduce.shuffle.max.connections\":\"0\",\"yarn.minicluster.yarn.nodemanager.resource.memory-mb\":\"4096\",\"mapreduce.job.emit-timeline-data\":\"false\",\"yarn.nodemanager.resource.system-reserved-memory-mb\":\"-1\",\"hadoop.kerberos.min.seconds.before.relogin\":\"60\",\"mapreduce.jobhistory.move.thread-count\":\"3\",\"yarn.resourcemanager.admin.client.thread-count\":\"1\",\"yarn.dispatcher.drain-events.timeout\":\"300000\",\"ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds\":\"10s,20s,30s,40s\",\"fs.s3a.buffer.dir\":\"${hadoop.tmp.dir}/s3a\",\"hadoop.ssl.enabled.protocols\":\"TLSv1.2\",\"mapreduce.jobhistory.admin.address\":\"0.0.0.0:10033\",\"yarn.log-aggregation-status.time-out.ms\":\"600000\",\"fs.s3a.accesspoint.required\":\"false\",\"mapreduce.shuffle.port\":\"13562\",\"yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory\":\"10\",\"yarn.nodemanager.health-checker.interval-ms\":\"600000\",\"yarn.resourcemanager.proxy.connection.timeout\":\"60000\",\"yarn.router.clientrm.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor\",\"yarn.resourcemanager.zk-appid-node.split-index\":\"0\",\"ftp.blocksize\":\"67108864\",\"yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions\":\"read\",\"yarn.router.rmadmin.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor\",\"yarn.nodemanager.log-container-debug-info.enabled\":\"true\",\"yarn.resourcemanager.activities-manager.app-activities.max-queue-length\":\"100\",\"yarn.resourcemanager.application-https.policy\":\"NONE\",\"yarn.client.max-cached-nodemanagers-proxies\":\"0\",\"yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms\":\"20\",\"yarn.nodemanager.delete.debug-delay-sec\":\"0\",\"yarn.nodemanager.pmem-check-enabled\":\"true\",\"yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage\":\"90.0\",\"mapreduce.app-submission.cross-platform\":\"false\",\"yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms\":\"10000\",\"yarn.nodemanager.container-retry-minimum-interval-ms\":\"1000\",\"hadoop.security.groups.cache.secs\":\"300\",\"yarn.federation.enabled\":\"false\",\"yarn.workflow-id.tag-prefix\":\"workflowid:\",\"fs.azure.local.sas.key.mode\":\"false\",\"ipc.maximum.data.length\":\"134217728\",\"fs.s3a.endpoint\":\"s3.amazonaws.com\",\"mapreduce.shuffle.max.threads\":\"0\",\"yarn.router.pipeline.cache-max-size\":\"25\",\"yarn.resourcemanager.nm-container-queuing.load-comparator\":\"QUEUE_LENGTH\",\"yarn.resourcemanager.resource-tracker.nm.ip-hostname-check\":\"false\",\"hadoop.security.authorization\":\"false\",\"mapreduce.job.complete.cancel.delegation.tokens\":\"*********(redacted)\",\"fs.s3a.paging.maximum\":\"5000\",\"nfs.exports.allowed.hosts\":\"* rw\",\"yarn.nodemanager.amrmproxy.ha.enable\":\"false\",\"fs.AbstractFileSystem.gs.impl\":\"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS\",\"mapreduce.jobhistory.http.policy\":\"HTTP_ONLY\",\"yarn.sharedcache.store.in-memory.check-period-mins\":\"720\",\"hadoop.security.group.mapping.ldap.ssl\":\"false\",\"fs.s3a.downgrade.syncable.exceptions\":\"true\",\"yarn.client.application-client-protocol.poll-interval-ms\":\"200\",\"yarn.scheduler.configuration.leveldb-store.compaction-interval-secs\":\"86400\",\"yarn.timeline-service.writer.class\":\"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl\",\"ha.zookeeper.parent-znode\":\"/hadoop-ha\",\"yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms\":\"60000\",\"yarn.nodemanager.log-aggregation.policy.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy\",\"mapreduce.reduce.shuffle.merge.percent\":\"0.66\",\"hadoop.security.group.mapping.ldap.search.filter.group\":\"(objectClass=group)\",\"yarn.resourcemanager.placement-constraints.scheduler.pool-size\":\"1\",\"yarn.resourcemanager.activities-manager.cleanup-interval-ms\":\"5000\",\"yarn.nodemanager.resourcemanager.minimum.version\":\"NONE\",\"mapreduce.job.speculative.speculative-cap-running-tasks\":\"0.1\",\"yarn.admin.acl\":\"*\",\"ipc.[port_number].identity-provider.impl\":\"org.apache.hadoop.ipc.UserIdentityProvider\",\"yarn.nodemanager.recovery.supervised\":\"false\",\"yarn.sharedcache.admin.thread-count\":\"1\",\"yarn.resourcemanager.ha.automatic-failover.enabled\":\"true\",\"yarn.nodemanager.container-log-monitor.total-size-limit-bytes\":\"10000000000\",\"mapreduce.reduce.skip.maxgroups\":\"0\",\"mapreduce.reduce.shuffle.connect.timeout\":\"180000\",\"yarn.nodemanager.health-checker.scripts\":\"script\",\"yarn.resourcemanager.address\":\"${yarn.resourcemanager.hostname}:8032\",\"ipc.client.ping\":\"true\",\"mapreduce.task.local-fs.write-limit.bytes\":\"-1\",\"fs.adl.oauth2.access.token.provider.type\":\"*********(redacted)\",\"mapreduce.shuffle.ssl.file.buffer.size\":\"65536\",\"yarn.resourcemanager.ha.automatic-failover.embedded\":\"true\",\"yarn.nodemanager.resource-plugins.gpu.docker-plugin\":\"nvidia-docker-v1\",\"fs.s3a.s3guard.consistency.retry.interval\":\"2s\",\"fs.s3a.multipart.purge\":\"false\",\"yarn.scheduler.configuration.store.class\":\"file\",\"yarn.resourcemanager.nm-container-queuing.queue-limit-stdev\":\"1.0f\",\"mapreduce.job.end-notification.max.attempts\":\"5\",\"mapreduce.output.fileoutputformat.compress.codec\":\"org.apache.hadoop.io.compress.DefaultCodec\",\"yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled\":\"false\",\"ipc.client.bind.wildcard.addr\":\"false\",\"yarn.resourcemanager.webapp.rest-csrf.enabled\":\"false\",\"ha.health-monitor.connect-retry-interval.ms\":\"1000\",\"yarn.nodemanager.keytab\":\"/etc/krb5.keytab\",\"mapreduce.jobhistory.keytab\":\"/etc/security/keytab/jhs.service.keytab\",\"fs.s3a.threads.max\":\"64\",\"yarn.nodemanager.runtime.linux.docker.image-update\":\"false\",\"mapreduce.reduce.shuffle.input.buffer.percent\":\"0.70\",\"fs.viewfs.overload.scheme.target.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem\",\"yarn.dispatcher.cpu-monitor.samples-per-min\":\"60\",\"hadoop.security.token.service.use_ip\":\"*********(redacted)\",\"yarn.nodemanager.runtime.linux.docker.allowed-container-networks\":\"host,none,bridge\",\"yarn.nodemanager.node-labels.resync-interval-ms\":\"120000\",\"hadoop.tmp.dir\":\"/tmp/hadoop-${user.name}\",\"mapreduce.job.maps\":\"2\",\"mapreduce.jobhistory.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.job.end-notification.max.retry.interval\":\"5000\",\"yarn.log-aggregation.retain-check-interval-seconds\":\"-1\",\"yarn.resourcemanager.resource-tracker.client.thread-count\":\"50\",\"yarn.nodemanager.containers-launcher.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher\",\"yarn.rm.system-metrics-publisher.emit-container-events\":\"false\",\"yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size\":\"10000\",\"yarn.resourcemanager.ha.automatic-failover.zk-base-path\":\"/yarn-leader-election\",\"io.seqfile.local.dir\":\"${hadoop.tmp.dir}/io/local\",\"fs.s3a.s3guard.ddb.throttle.retry.interval\":\"100ms\",\"fs.AbstractFileSystem.wasb.impl\":\"org.apache.hadoop.fs.azure.Wasb\",\"mapreduce.client.submit.file.replication\":\"10\",\"mapreduce.jobhistory.minicluster.fixed.ports\":\"false\",\"fs.s3a.multipart.threshold\":\"128M\",\"yarn.resourcemanager.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"mapreduce.jobhistory.done-dir\":\"${yarn.app.mapreduce.am.staging-dir}/history/done\",\"ipc.server.purge.interval\":\"15\",\"ipc.client.idlethreshold\":\"4000\",\"yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage\":\"false\",\"mapreduce.reduce.input.buffer.percent\":\"0.0\",\"yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold\":\"1\",\"yarn.nodemanager.webapp.rest-csrf.enabled\":\"false\",\"fs.ftp.host.port\":\"21\",\"ipc.ping.interval\":\"60000\",\"yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size\":\"10\",\"yarn.resourcemanager.admin.address\":\"${yarn.resourcemanager.hostname}:8033\",\"file.client-write-packet-size\":\"65536\",\"ipc.client.kill.max\":\"10\",\"mapreduce.reduce.speculative\":\"true\",\"hadoop.security.key.default.bitlength\":\"128\",\"mapreduce.job.reducer.unconditional-preempt.delay.sec\":\"300\",\"yarn.nodemanager.disk-health-checker.interval-ms\":\"120000\",\"yarn.nodemanager.log.deletion-threads-count\":\"4\",\"fs.s3a.committer.abort.pending.uploads\":\"true\",\"yarn.webapp.filter-entity-list-by-user\":\"false\",\"yarn.resourcemanager.activities-manager.app-activities.ttl-ms\":\"600000\",\"ipc.client.connection.maxidletime\":\"10000\",\"mapreduce.task.io.sort.mb\":\"100\",\"yarn.nodemanager.localizer.client.thread-count\":\"5\",\"io.erasurecode.codec.rs.rawcoders\":\"rs_native,rs_java\",\"io.erasurecode.codec.rs-legacy.rawcoders\":\"rs-legacy_java\",\"yarn.sharedcache.admin.address\":\"0.0.0.0:8047\",\"yarn.resourcemanager.placement-constraints.algorithm.iterator\":\"SERIAL\",\"yarn.nodemanager.localizer.cache.cleanup.interval-ms\":\"600000\",\"hadoop.security.crypto.codec.classes.aes.ctr.nopadding\":\"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec\",\"mapreduce.job.cache.limit.max-resources-mb\":\"0\",\"fs.s3a.connection.ssl.enabled\":\"true\",\"yarn.nodemanager.process-kill-wait.ms\":\"5000\",\"mapreduce.job.hdfs-servers\":\"${fs.defaultFS}\",\"yarn.app.mapreduce.am.webapp.https.client.auth\":\"false\",\"hadoop.workaround.non.threadsafe.getpwuid\":\"true\",\"fs.df.interval\":\"60000\",\"ipc.[port_number].decay-scheduler.thresholds\":\"13,25,50\",\"fs.s3a.multiobjectdelete.enable\":\"true\",\"yarn.sharedcache.cleaner.resource-sleep-ms\":\"0\",\"yarn.nodemanager.disk-health-checker.min-healthy-disks\":\"0.25\",\"hadoop.shell.missing.defaultFs.warning\":\"false\",\"io.file.buffer.size\":\"65536\",\"fs.viewfs.overload.scheme.target.wasb.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem\",\"hadoop.security.group.mapping.ldap.search.attr.member\":\"member\",\"hadoop.security.random.device.file.path\":\"/dev/urandom\",\"hadoop.security.sensitive-config-keys\":\"*********(redacted)\",\"fs.s3a.s3guard.ddb.max.retries\":\"9\",\"fs.viewfs.overload.scheme.target.file.impl\":\"org.apache.hadoop.fs.LocalFileSystem\",\"hadoop.rpc.socket.factory.class.default\":\"org.apache.hadoop.net.StandardSocketFactory\",\"yarn.intermediate-data-encryption.enable\":\"false\",\"yarn.resourcemanager.connect.retry-interval.ms\":\"30000\",\"yarn.nodemanager.container.stderr.pattern\":\"{*stderr*,*STDERR*}\",\"yarn.scheduler.minimum-allocation-mb\":\"1024\",\"yarn.app.mapreduce.am.staging-dir\":\"/tmp/hadoop-yarn/staging\",\"mapreduce.reduce.shuffle.read.timeout\":\"180000\",\"hadoop.http.cross-origin.max-age\":\"1800\",\"io.erasurecode.codec.xor.rawcoders\":\"xor_native,xor_java\",\"fs.s3a.s3guard.consistency.retry.limit\":\"7\",\"fs.s3a.connection.establish.timeout\":\"5000\",\"mapreduce.job.running.map.limit\":\"0\",\"yarn.minicluster.control-resource-monitoring\":\"false\",\"hadoop.ssl.require.client.cert\":\"false\",\"hadoop.kerberos.kinit.command\":\"kinit\",\"yarn.federation.state-store.class\":\"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore\",\"mapreduce.reduce.log.level\":\"INFO\",\"hadoop.security.dns.log-slow-lookups.threshold.ms\":\"1000\",\"mapreduce.job.ubertask.enable\":\"false\",\"adl.http.timeout\":\"-1\",\"yarn.resourcemanager.placement-constraints.retry-attempts\":\"3\",\"hadoop.caller.context.enabled\":\"false\",\"hadoop.security.group.mapping.ldap.num.attempts\":\"3\",\"yarn.nodemanager.vmem-pmem-ratio\":\"2.1\",\"hadoop.rpc.protection\":\"authentication\",\"ha.health-monitor.rpc-timeout.ms\":\"45000\",\"yarn.nodemanager.remote-app-log-dir\":\"/tmp/logs\",\"hadoop.zk.timeout-ms\":\"10000\",\"fs.s3a.s3guard.cli.prune.age\":\"86400000\",\"yarn.nodemanager.resource.pcores-vcores-multiplier\":\"1.0\",\"yarn.nodemanager.runtime.linux.sandbox-mode\":\"disabled\",\"yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size\":\"10\",\"fs.viewfs.overload.scheme.target.webhdfs.impl\":\"org.apache.hadoop.hdfs.web.WebHdfsFileSystem\",\"fs.s3a.committer.threads\":\"8\",\"hadoop.zk.retry-interval-ms\":\"1000\",\"hadoop.security.crypto.buffer.size\":\"8192\",\"yarn.nodemanager.node-labels.provider.fetch-interval-ms\":\"600000\",\"mapreduce.jobhistory.recovery.store.leveldb.path\":\"${hadoop.tmp.dir}/mapred/history/recoverystore\",\"yarn.client.failover-retries-on-socket-timeouts\":\"0\",\"fs.s3a.ssl.channel.mode\":\"default_jsse\",\"yarn.nodemanager.resource.memory.enabled\":\"false\",\"fs.azure.authorization.caching.enable\":\"true\",\"hadoop.security.instrumentation.requires.admin\":\"false\",\"yarn.nodemanager.delete.thread-count\":\"4\",\"mapreduce.job.finish-when-all-reducers-done\":\"true\",\"hadoop.registry.jaas.context\":\"Client\",\"yarn.timeline-service.leveldb-timeline-store.path\":\"${hadoop.tmp.dir}/yarn/timeline\",\"io.map.index.interval\":\"128\",\"yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms\":\"100\",\"fs.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem\",\"mapreduce.job.counters.max\":\"120\",\"mapreduce.jobhistory.webapp.rest-csrf.enabled\":\"false\",\"yarn.timeline-service.store-class\":\"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore\",\"mapreduce.jobhistory.move.interval-ms\":\"180000\",\"fs.s3a.change.detection.version.required\":\"true\",\"yarn.nodemanager.localizer.fetch.thread-count\":\"4\",\"yarn.resourcemanager.scheduler.client.thread-count\":\"50\",\"hadoop.ssl.hostname.verifier\":\"DEFAULT\",\"yarn.timeline-service.leveldb-state-store.path\":\"${hadoop.tmp.dir}/yarn/timeline\",\"mapreduce.job.classloader\":\"false\",\"mapreduce.task.profile.map.params\":\"${mapreduce.task.profile.params}\",\"ipc.client.connect.timeout\":\"20000\",\"hadoop.security.auth_to_local.mechanism\":\"hadoop\",\"yarn.timeline-service.app-collector.linger-period.ms\":\"60000\",\"yarn.nm.liveness-monitor.expiry-interval-ms\":\"600000\",\"yarn.resourcemanager.reservation-system.planfollower.time-step\":\"1000\",\"yarn.resourcemanager.proxy.timeout.enabled\":\"true\",\"yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms\":\"600000\",\"yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed\":\"true\",\"yarn.webapp.api-service.enable\":\"false\",\"yarn.nodemanager.recovery.enabled\":\"false\",\"mapreduce.job.end-notification.retry.interval\":\"1000\",\"fs.du.interval\":\"600000\",\"fs.ftp.impl\":\"org.apache.hadoop.fs.ftp.FTPFileSystem\",\"yarn.nodemanager.container.stderr.tail.bytes\":\"4096\",\"yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled\":\"true\",\"hadoop.security.group.mapping.ldap.read.timeout.ms\":\"60000\",\"hadoop.security.groups.cache.warn.after.ms\":\"5000\",\"file.bytes-per-checksum\":\"512\",\"mapreduce.outputcommitter.factory.scheme.s3a\":\"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory\",\"hadoop.security.groups.cache.background.reload\":\"false\",\"yarn.nodemanager.container-monitor.enabled\":\"true\",\"yarn.nodemanager.elastic-memory-control.enabled\":\"false\",\"net.topology.script.number.args\":\"100\",\"mapreduce.task.merge.progress.records\":\"10000\",\"yarn.nodemanager.localizer.address\":\"${yarn.nodemanager.hostname}:8040\",\"yarn.timeline-service.keytab\":\"/etc/krb5.keytab\",\"mapreduce.reduce.shuffle.fetch.retry.timeout-ms\":\"30000\",\"yarn.resourcemanager.rm.container-allocation.expiry-interval-ms\":\"600000\",\"yarn.nodemanager.container-executor.exit-code-file.timeout-ms\":\"2000\",\"mapreduce.fileoutputcommitter.algorithm.version\":\"1\",\"yarn.resourcemanager.work-preserving-recovery.enabled\":\"true\",\"mapreduce.map.skip.maxrecords\":\"0\",\"yarn.sharedcache.root-dir\":\"/sharedcache\",\"fs.s3a.retry.throttle.limit\":\"20\",\"hadoop.http.authentication.type\":\"simple\",\"fs.viewfs.overload.scheme.target.oss.impl\":\"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem\",\"mapreduce.job.cache.limit.max-resources\":\"0\",\"mapreduce.task.userlog.limit.kb\":\"0\",\"ipc.[port_number].weighted-cost.handler\":\"1\",\"yarn.resourcemanager.scheduler.monitor.enable\":\"false\",\"ipc.client.connect.max.retries\":\"10\",\"hadoop.registry.zk.retry.times\":\"5\",\"yarn.nodemanager.resource-monitor.interval-ms\":\"3000\",\"yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices\":\"auto\",\"mapreduce.job.sharedcache.mode\":\"disabled\",\"yarn.nodemanager.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.shuffle.listen.queue.size\":\"128\",\"yarn.scheduler.configuration.mutation.acl-policy.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy\",\"mapreduce.map.cpu.vcores\":\"1\",\"yarn.log-aggregation.file-formats\":\"TFile\",\"yarn.timeline-service.client.fd-retain-secs\":\"300\",\"fs.s3a.select.output.csv.field.delimiter\":\",\",\"yarn.nodemanager.health-checker.timeout-ms\":\"1200000\",\"hadoop.user.group.static.mapping.overrides\":\"dr.who=;\",\"fs.azure.sas.expiry.period\":\"90d\",\"fs.s3a.select.output.csv.record.delimiter\":\"\\\\n\",\"mapreduce.jobhistory.recovery.store.class\":\"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService\",\"fs.viewfs.overload.scheme.target.https.impl\":\"org.apache.hadoop.fs.http.HttpsFileSystem\",\"fs.s3a.s3guard.ddb.table.sse.enabled\":\"false\",\"yarn.resourcemanager.fail-fast\":\"${yarn.fail-fast}\",\"yarn.resourcemanager.proxy-user-privileges.enabled\":\"false\",\"yarn.router.webapp.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST\",\"yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage\":\"90.0\",\"mapreduce.job.reducer.preempt.delay.sec\":\"0\",\"hadoop.util.hash.type\":\"murmur\",\"yarn.nodemanager.disk-validator\":\"basic\",\"yarn.app.mapreduce.client.job.max-retries\":\"3\",\"fs.viewfs.overload.scheme.target.ftp.impl\":\"org.apache.hadoop.fs.ftp.FTPFileSystem\",\"mapreduce.reduce.shuffle.retry-delay.max.ms\":\"60000\",\"hadoop.security.group.mapping.ldap.connection.timeout.ms\":\"60000\",\"mapreduce.task.profile.params\":\"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s\",\"yarn.app.mapreduce.shuffle.log.backups\":\"0\",\"yarn.nodemanager.container-diagnostics-maximum-size\":\"10000\",\"hadoop.registry.zk.retry.interval.ms\":\"1000\",\"yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms\":\"1000\",\"fs.AbstractFileSystem.file.impl\":\"org.apache.hadoop.fs.local.LocalFs\",\"yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds\":\"-1\",\"mapreduce.jobhistory.cleaner.interval-ms\":\"86400000\",\"hadoop.registry.zk.quorum\":\"localhost:2181\",\"yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes\":\"runc\",\"mapreduce.output.fileoutputformat.compress\":\"false\",\"yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"fs.s3a.assumed.role.session.duration\":\"30m\",\"hadoop.security.group.mapping.ldap.conversion.rule\":\"none\",\"hadoop.ssl.server.conf\":\"ssl-server.xml\",\"fs.s3a.retry.throttle.interval\":\"100ms\",\"seq.io.sort.factor\":\"100\",\"fs.viewfs.overload.scheme.target.ofs.impl\":\"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem\",\"yarn.sharedcache.cleaner.initial-delay-mins\":\"10\",\"mapreduce.client.completion.pollinterval\":\"5000\",\"hadoop.ssl.keystores.factory.class\":\"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory\",\"yarn.app.mapreduce.am.resource.cpu-vcores\":\"1\",\"yarn.timeline-service.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.docker.capabilities\":\"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE\",\"yarn.acl.enable\":\"false\",\"yarn.timeline-service.entity-group-fs-store.done-dir\":\"/tmp/entity-file-history/done/\",\"hadoop.security.group.mapping.ldap.num.attempts.before.failover\":\"3\",\"mapreduce.task.profile\":\"false\",\"hadoop.prometheus.endpoint.enabled\":\"false\",\"yarn.resourcemanager.fs.state-store.uri\":\"${hadoop.tmp.dir}/yarn/system/rmstore\",\"mapreduce.jobhistory.always-scan-user-dir\":\"false\",\"fs.s3a.metadatastore.metadata.ttl\":\"15m\",\"yarn.nodemanager.opportunistic-containers-use-pause-for-preemption\":\"false\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user\":\"nobody\",\"yarn.timeline-service.reader.class\":\"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl\",\"yarn.resourcemanager.configuration.provider-class\":\"org.apache.hadoop.yarn.LocalConfigurationProvider\",\"yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold\":\"1\",\"yarn.resourcemanager.configuration.file-system-based-store\":\"/yarn/conf\",\"mapreduce.job.cache.limit.max-single-resource-mb\":\"0\",\"yarn.nodemanager.runtime.linux.docker.stop.grace-period\":\"10\",\"yarn.resourcemanager.resource-profiles.source-file\":\"resource-profiles.json\",\"mapreduce.job.dfs.storage.capacity.kill-limit-exceed\":\"false\",\"yarn.nodemanager.resource.percentage-physical-cpu-limit\":\"100\",\"mapreduce.jobhistory.client.thread-count\":\"10\",\"tfile.fs.input.buffer.size\":\"262144\",\"mapreduce.client.progressmonitor.pollinterval\":\"1000\",\"yarn.nodemanager.log-dirs\":\"${yarn.log.dir}/userlogs\",\"yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat\":\"-1\",\"fs.automatic.close\":\"true\",\"yarn.resourcemanager.delegation-token-renewer.thread-retry-interval\":\"*********(redacted)\",\"fs.s3a.select.input.csv.quote.character\":\"\\\"\",\"yarn.nodemanager.hostname\":\"0.0.0.0\",\"ipc.[port_number].cost-provider.impl\":\"org.apache.hadoop.ipc.DefaultCostProvider\",\"yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin\",\"yarn.nodemanager.remote-app-log-dir-include-older\":\"true\",\"yarn.nodemanager.resource.memory.cgroups.swappiness\":\"0\",\"ftp.stream-buffer-size\":\"4096\",\"yarn.fail-fast\":\"false\",\"yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep\":\"100\",\"yarn.timeline-service.app-aggregation-interval-secs\":\"15\",\"hadoop.security.group.mapping.ldap.search.filter.user\":\"(&(objectClass=user)(sAMAccountName={0}))\",\"ipc.[port_number].weighted-cost.lockshared\":\"10\",\"yarn.nodemanager.container-localizer.log.level\":\"INFO\",\"yarn.timeline-service.address\":\"${yarn.timeline-service.hostname}:10200\",\"mapreduce.job.ubertask.maxmaps\":\"9\",\"fs.s3a.threads.keepalivetime\":\"60\",\"mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"mapreduce.task.files.preserve.failedtasks\":\"false\",\"yarn.app.mapreduce.client.job.retry-interval\":\"2000\",\"ha.failover-controller.graceful-fence.connection.retries\":\"1\",\"fs.s3a.select.output.csv.quote.escape.character\":\"\\\\\\\\\",\"yarn.resourcemanager.delegation.token.max-lifetime\":\"*********(redacted)\",\"hadoop.kerberos.keytab.login.autorenewal.enabled\":\"false\",\"yarn.timeline-service.client.drain-entities.timeout.ms\":\"2000\",\"yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms\":\"1000\",\"yarn.timeline-service.entity-group-fs-store.summary-store\":\"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore\",\"mapreduce.reduce.cpu.vcores\":\"1\",\"mapreduce.job.encrypted-intermediate-data.buffer.kb\":\"128\",\"fs.client.resolve.remote.symlinks\":\"true\",\"yarn.nodemanager.webapp.https.address\":\"0.0.0.0:8044\",\"hadoop.http.cross-origin.allowed-origins\":\"*\",\"mapreduce.job.encrypted-intermediate-data\":\"false\",\"yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled\":\"true\",\"fs.s3a.executor.capacity\":\"16\",\"yarn.timeline-service.entity-group-fs-store.retain-seconds\":\"604800\",\"yarn.resourcemanager.metrics.runtime.buckets\":\"60,300,1440\",\"yarn.timeline-service.generic-application-history.max-applications\":\"10000\",\"yarn.nodemanager.local-dirs\":\"${hadoop.tmp.dir}/nm-local-dir\",\"mapreduce.shuffle.connection-keep-alive.enable\":\"false\",\"yarn.node-labels.configuration-type\":\"centralized\",\"fs.s3a.path.style.access\":\"false\",\"yarn.nodemanager.aux-services.mapreduce_shuffle.class\":\"org.apache.hadoop.mapred.ShuffleHandler\",\"yarn.sharedcache.store.in-memory.staleness-period-mins\":\"10080\",\"fs.adl.impl\":\"org.apache.hadoop.fs.adl.AdlFileSystem\",\"yarn.resourcemanager.application.max-tags\":\"10\",\"hadoop.domainname.resolver.impl\":\"org.apache.hadoop.net.DNSDomainNameResolver\",\"yarn.resourcemanager.nodemanager.minimum.version\":\"NONE\",\"mapreduce.jobhistory.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled\":\"false\",\"net.topology.impl\":\"org.apache.hadoop.net.NetworkTopology\",\"io.map.index.skip\":\"0\",\"yarn.timeline-service.reader.webapp.https.address\":\"${yarn.timeline-service.webapp.https.address}\",\"fs.ftp.data.connection.mode\":\"ACTIVE_LOCAL_DATA_CONNECTION_MODE\",\"mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed\":\"true\",\"fs.azure.buffer.dir\":\"${hadoop.tmp.dir}/abfs\",\"yarn.scheduler.maximum-allocation-vcores\":\"4\",\"hadoop.http.cross-origin.allowed-headers\":\"X-Requested-With,Content-Type,Accept,Origin\",\"yarn.nodemanager.log-aggregation.compression-type\":\"none\",\"yarn.timeline-service.version\":\"1.0f\",\"yarn.ipc.rpc.class\":\"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC\",\"mapreduce.reduce.maxattempts\":\"4\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size\":\"1000\",\"hadoop.security.dns.log-slow-lookups.enabled\":\"false\",\"mapreduce.job.committer.setup.cleanup.needed\":\"true\",\"hadoop.security.secure.random.impl\":\"org.apache.hadoop.crypto.random.OpensslSecureRandom\",\"mapreduce.job.running.reduce.limit\":\"0\",\"fs.s3a.select.errors.include.sql\":\"false\",\"fs.s3a.connection.request.timeout\":\"0\",\"ipc.maximum.response.length\":\"134217728\",\"yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"mapreduce.job.token.tracking.ids.enabled\":\"*********(redacted)\",\"hadoop.caller.context.max.size\":\"128\",\"yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed\":\"false\",\"yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed\":\"false\",\"hadoop.registry.system.acls\":\"sasl:yarn@, sasl:mapred@, sasl:hdfs@\",\"yarn.nodemanager.recovery.dir\":\"${hadoop.tmp.dir}/yarn-nm-recovery\",\"fs.s3a.fast.upload.buffer\":\"disk\",\"mapreduce.jobhistory.intermediate-done-dir\":\"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate\",\"yarn.app.mapreduce.shuffle.log.separate\":\"true\",\"yarn.log-aggregation.debug.filesize\":\"104857600\",\"fs.s3a.max.total.tasks\":\"32\",\"fs.s3a.readahead.range\":\"64K\",\"hadoop.http.authentication.simple.anonymous.allowed\":\"true\",\"fs.s3a.attempts.maximum\":\"20\",\"hadoop.registry.zk.connection.timeout.ms\":\"15000\",\"yarn.resourcemanager.delegation-token-renewer.thread-count\":\"*********(redacted)\",\"yarn.resourcemanager.delegation-token-renewer.thread-timeout\":\"*********(redacted)\",\"yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size\":\"10000\",\"yarn.nodemanager.aux-services.manifest.reload-ms\":\"0\",\"yarn.nodemanager.emit-container-events\":\"true\",\"yarn.resourcemanager.resource-profiles.enabled\":\"false\",\"yarn.timeline-service.hbase-schema.prefix\":\"prod.\",\"fs.azure.authorization\":\"false\",\"mapreduce.map.log.level\":\"INFO\",\"ha.failover-controller.active-standby-elector.zk.op.retries\":\"3\",\"yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs\":\"20\",\"mapreduce.output.fileoutputformat.compress.type\":\"RECORD\",\"yarn.resourcemanager.leveldb-state-store.path\":\"${hadoop.tmp.dir}/yarn/system/rmstore\",\"yarn.timeline-service.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.ifile.readahead.bytes\":\"4194304\",\"yarn.sharedcache.app-checker.class\":\"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users\":\"true\",\"yarn.nodemanager.resource.detect-hardware-capabilities\":\"false\",\"mapreduce.cluster.acls.enabled\":\"false\",\"mapreduce.job.speculative.retry-after-no-speculate\":\"1000\",\"fs.viewfs.overload.scheme.target.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem\",\"hadoop.security.group.mapping.ldap.search.group.hierarchy.levels\":\"0\",\"yarn.resourcemanager.fs.state-store.retry-interval-ms\":\"1000\",\"file.stream-buffer-size\":\"4096\",\"yarn.resourcemanager.application-timeouts.monitor.interval-ms\":\"3000\",\"mapreduce.map.output.compress.codec\":\"org.apache.hadoop.io.compress.DefaultCodec\",\"mapreduce.map.speculative\":\"true\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file\":\"/runc-root/image-tag-to-hash\",\"mapreduce.job.speculative.retry-after-speculate\":\"15000\",\"yarn.nodemanager.linux-container-executor.cgroups.mount\":\"false\",\"yarn.app.mapreduce.am.container.log.backups\":\"0\",\"yarn.app.mapreduce.am.log.level\":\"INFO\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin\",\"io.bytes.per.checksum\":\"512\",\"mapreduce.job.reduce.slowstart.completedmaps\":\"0.05\",\"yarn.timeline-service.http-authentication.type\":\"simple\",\"hadoop.security.group.mapping.ldap.search.attr.group.name\":\"cn\",\"yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices\":\"auto\",\"yarn.timeline-service.client.internal-timers-ttl-secs\":\"420\",\"fs.s3a.select.output.csv.quote.character\":\"\\\"\",\"hadoop.http.logs.enabled\":\"true\",\"fs.s3a.block.size\":\"32M\",\"yarn.sharedcache.client-server.address\":\"0.0.0.0:8045\",\"yarn.nodemanager.logaggregation.threadpool-size-max\":\"100\",\"yarn.resourcemanager.hostname\":\"0.0.0.0\",\"yarn.resourcemanager.delegation.key.update-interval\":\"86400000\",\"mapreduce.reduce.shuffle.fetch.retry.enabled\":\"${yarn.nodemanager.recovery.enabled}\",\"mapreduce.map.memory.mb\":\"-1\",\"mapreduce.task.skip.start.attempts\":\"2\",\"fs.AbstractFileSystem.hdfs.impl\":\"org.apache.hadoop.fs.Hdfs\",\"yarn.nodemanager.disk-health-checker.enable\":\"true\",\"fs.s3a.select.output.csv.quote.fields\":\"always\",\"ipc.client.tcpnodelay\":\"true\",\"ipc.client.rpc-timeout.ms\":\"0\",\"yarn.nodemanager.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts\":\"*********(redacted)\",\"ipc.client.low-latency\":\"false\",\"mapreduce.input.lineinputformat.linespermap\":\"1\",\"yarn.router.interceptor.user.threadpool-size\":\"5\",\"ipc.client.connect.max.retries.on.timeouts\":\"45\",\"yarn.timeline-service.leveldb-timeline-store.read-cache-size\":\"104857600\",\"fs.AbstractFileSystem.har.impl\":\"org.apache.hadoop.fs.HarFs\",\"mapreduce.job.split.metainfo.maxsize\":\"10000000\",\"yarn.am.liveness-monitor.expiry-interval-ms\":\"600000\",\"yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"yarn.timeline-service.entity-group-fs-store.app-cache-size\":\"10\",\"yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs\":\"360\",\"fs.s3a.socket.recv.buffer\":\"8192\",\"rpc.metrics.timeunit\":\"MILLISECONDS\",\"yarn.resourcemanager.resource-tracker.address\":\"${yarn.resourcemanager.hostname}:8031\",\"yarn.nodemanager.node-labels.provider.fetch-timeout-ms\":\"1200000\",\"mapreduce.job.heap.memory-mb.ratio\":\"0.8\",\"yarn.resourcemanager.leveldb-state-store.compaction-interval-secs\":\"3600\",\"yarn.resourcemanager.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"yarn.nodemanager.pluggable-device-framework.enabled\":\"false\",\"yarn.scheduler.configuration.fs.path\":\"file://${hadoop.tmp.dir}/yarn/system/schedconf\",\"mapreduce.client.output.filter\":\"FAILED\",\"hadoop.http.filter.initializers\":\"org.apache.hadoop.http.lib.StaticUserWebFilter\",\"mapreduce.reduce.memory.mb\":\"-1\",\"yarn.timeline-service.hostname\":\"0.0.0.0\",\"file.replication\":\"1\",\"yarn.nodemanager.container-metrics.unregister-delay-ms\":\"10000\",\"yarn.nodemanager.container-metrics.period-ms\":\"-1\",\"mapreduce.fileoutputcommitter.task.cleanup.enabled\":\"false\",\"yarn.nodemanager.log.retain-seconds\":\"10800\",\"yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds\":\"3600\",\"ipc.[port_number].callqueue.impl\":\"java.util.concurrent.LinkedBlockingQueue\",\"yarn.resourcemanager.keytab\":\"/etc/krb5.keytab\",\"hadoop.security.group.mapping.providers.combined\":\"true\",\"mapreduce.reduce.merge.inmem.threshold\":\"1000\",\"yarn.timeline-service.recovery.enabled\":\"false\",\"fs.azure.saskey.usecontainersaskeyforallaccess\":\"true\",\"yarn.sharedcache.nm.uploader.thread-count\":\"20\",\"yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs\":\"3600\",\"ipc.[port_number].weighted-cost.lockfree\":\"1\",\"mapreduce.shuffle.ssl.enabled\":\"false\",\"yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds\":\"259200000\",\"yarn.nodemanager.opportunistic-containers-max-queue-length\":\"0\",\"yarn.resourcemanager.state-store.max-completed-applications\":\"${yarn.resourcemanager.max-completed-applications}\",\"mapreduce.job.speculative.minimum-allowed-tasks\":\"10\",\"fs.s3a.aws.credentials.provider\":\"\\n org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\\n org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\\n com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\\n org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\\n \",\"yarn.log-aggregation.retain-seconds\":\"-1\",\"yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb\":\"0\",\"mapreduce.jobhistory.max-age-ms\":\"604800000\",\"hadoop.http.cross-origin.allowed-methods\":\"GET,POST,HEAD\",\"yarn.resourcemanager.opportunistic-container-allocation.enabled\":\"false\",\"mapreduce.jobhistory.webapp.address\":\"0.0.0.0:19888\",\"hadoop.system.tags\":\"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL\",\"yarn.log-aggregation.file-controller.TFile.class\":\"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController\",\"yarn.client.nodemanager-connect.max-wait-ms\":\"180000\",\"yarn.resourcemanager.webapp.address\":\"${yarn.resourcemanager.hostname}:8088\",\"mapreduce.jobhistory.recovery.enable\":\"false\",\"mapreduce.reduce.shuffle.parallelcopies\":\"5\",\"fs.AbstractFileSystem.webhdfs.impl\":\"org.apache.hadoop.fs.WebHdfs\",\"fs.trash.interval\":\"0\",\"yarn.app.mapreduce.client.max-retries\":\"3\",\"hadoop.security.authentication\":\"simple\",\"mapreduce.task.profile.reduce.params\":\"${mapreduce.task.profile.params}\",\"yarn.app.mapreduce.am.resource.mb\":\"1536\",\"mapreduce.input.fileinputformat.list-status.num-threads\":\"1\",\"yarn.nodemanager.container-executor.class\":\"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor\",\"io.mapfile.bloom.size\":\"1048576\",\"yarn.timeline-service.ttl-ms\":\"604800000\",\"yarn.resourcemanager.nm-container-queuing.min-queue-length\":\"5\",\"yarn.nodemanager.resource.cpu-vcores\":\"-1\",\"mapreduce.job.reduces\":\"1\",\"fs.s3a.multipart.size\":\"64M\",\"fs.s3a.select.input.csv.comment.marker\":\"#\",\"yarn.scheduler.minimum-allocation-vcores\":\"1\",\"mapreduce.job.speculative.speculative-cap-total-tasks\":\"0.01\",\"hadoop.ssl.client.conf\":\"ssl-client.xml\",\"mapreduce.job.queuename\":\"default\",\"mapreduce.job.encrypted-intermediate-data-key-size-bits\":\"128\",\"fs.s3a.metadatastore.authoritative\":\"false\",\"ipc.[port_number].weighted-cost.response\":\"1\",\"yarn.nodemanager.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"ha.health-monitor.sleep-after-disconnect.ms\":\"1000\",\"yarn.app.mapreduce.shuffle.log.limit.kb\":\"0\",\"hadoop.security.group.mapping\":\"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback\",\"yarn.client.application-client-protocol.poll-timeout-ms\":\"-1\",\"mapreduce.jobhistory.jhist.format\":\"binary\",\"mapreduce.task.stuck.timeout-ms\":\"600000\",\"yarn.resourcemanager.application.max-tag.length\":\"100\",\"yarn.resourcemanager.ha.enabled\":\"false\",\"dfs.client.ignore.namenode.default.kms.uri\":\"false\",\"hadoop.http.staticuser.user\":\"dr.who\",\"mapreduce.task.exit.timeout.check-interval-ms\":\"20000\",\"mapreduce.jobhistory.intermediate-user-done-dir.permissions\":\"770\",\"mapreduce.task.exit.timeout\":\"60000\",\"yarn.nodemanager.linux-container-executor.resources-handler.class\":\"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler\",\"mapreduce.reduce.shuffle.memory.limit.percent\":\"0.25\",\"yarn.resourcemanager.reservation-system.enable\":\"false\",\"mapreduce.map.output.compress\":\"false\",\"ha.zookeeper.acl\":\"world:anyone:rwcda\",\"ipc.server.max.connections\":\"0\",\"yarn.nodemanager.runtime.linux.docker.default-container-network\":\"host\",\"yarn.router.webapp.address\":\"0.0.0.0:8089\",\"yarn.scheduler.maximum-allocation-mb\":\"8192\",\"yarn.resourcemanager.scheduler.monitor.policies\":\"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy\",\"yarn.sharedcache.cleaner.period-mins\":\"1440\",\"yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint\":\"http://localhost:3476/v1.0/docker/cli\",\"yarn.app.mapreduce.am.container.log.limit.kb\":\"0\",\"ipc.client.connect.retry.interval\":\"1000\",\"yarn.timeline-service.http-cross-origin.enabled\":\"false\",\"fs.wasbs.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms\":\"1000\",\"yarn.federation.subcluster-resolver.class\":\"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl\",\"yarn.resourcemanager.zk-state-store.parent-path\":\"/rmstore\",\"fs.s3a.select.input.csv.field.delimiter\":\",\",\"mapreduce.jobhistory.cleaner.enable\":\"true\",\"yarn.timeline-service.client.fd-flush-interval-secs\":\"10\",\"hadoop.security.kms.client.encrypted.key.cache.expiry\":\"43200000\",\"yarn.client.nodemanager-client-async.thread-pool-max-size\":\"500\",\"mapreduce.map.maxattempts\":\"4\",\"yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms\":\"1000\",\"fs.s3a.committer.staging.tmp.path\":\"tmp/staging\",\"yarn.nodemanager.sleep-delay-before-sigkill.ms\":\"250\",\"yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms\":\"10\",\"mapreduce.job.end-notification.retry.attempts\":\"0\",\"yarn.nodemanager.resource.count-logical-processors-as-cores\":\"false\",\"hadoop.registry.zk.root\":\"/registry\",\"adl.feature.ownerandgroup.enableupn\":\"false\",\"yarn.resourcemanager.zk-max-znode-size.bytes\":\"1048576\",\"mapreduce.job.reduce.shuffle.consumer.plugin.class\":\"org.apache.hadoop.mapreduce.task.reduce.Shuffle\",\"yarn.resourcemanager.delayed.delegation-token.removal-interval-ms\":\"*********(redacted)\",\"yarn.nodemanager.localizer.cache.target-size-mb\":\"10240\",\"fs.s3a.committer.staging.conflict-mode\":\"append\",\"mapreduce.client.libjars.wildcard\":\"true\",\"fs.s3a.committer.staging.unique-filenames\":\"true\",\"yarn.nodemanager.node-attributes.provider.fetch-timeout-ms\":\"1200000\",\"fs.s3a.list.version\":\"2\",\"ftp.client-write-packet-size\":\"65536\",\"ipc.[port_number].weighted-cost.lockexclusive\":\"100\",\"fs.AbstractFileSystem.adl.impl\":\"org.apache.hadoop.fs.adl.Adl\",\"yarn.nodemanager.container-log-monitor.enable\":\"false\",\"hadoop.security.key.default.cipher\":\"AES/CTR/NoPadding\",\"yarn.client.failover-retries\":\"0\",\"fs.s3a.multipart.purge.age\":\"86400\",\"mapreduce.job.local-fs.single-disk-limit.check.interval-ms\":\"5000\",\"net.topology.node.switch.mapping.impl\":\"org.apache.hadoop.net.ScriptBasedMapping\",\"yarn.nodemanager.amrmproxy.address\":\"0.0.0.0:8049\",\"ipc.server.listen.queue.size\":\"256\",\"ipc.[port_number].decay-scheduler.period-ms\":\"5000\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs\":\"60\",\"map.sort.class\":\"org.apache.hadoop.util.QuickSort\",\"fs.viewfs.rename.strategy\":\"SAME_MOUNTPOINT\",\"hadoop.security.kms.client.authentication.retry-count\":\"1\",\"fs.permissions.umask-mode\":\"022\",\"fs.s3a.assumed.role.credentials.provider\":\"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\",\"yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed\":\"false\",\"yarn.nodemanager.vmem-check-enabled\":\"true\",\"yarn.nodemanager.numa-awareness.enabled\":\"false\",\"yarn.nodemanager.recovery.compaction-interval-secs\":\"3600\",\"yarn.app.mapreduce.client-am.ipc.max-retries\":\"3\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds\":\"60\",\"yarn.federation.registry.base-dir\":\"yarnfederation/\",\"yarn.nodemanager.health-checker.run-before-startup\":\"false\",\"mapreduce.job.max.map\":\"-1\",\"mapreduce.job.local-fs.single-disk-limit.bytes\":\"-1\",\"mapreduce.shuffle.pathcache.concurrency-level\":\"16\",\"mapreduce.job.ubertask.maxreduces\":\"1\",\"mapreduce.shuffle.pathcache.max-weight\":\"10485760\",\"hadoop.security.kms.client.encrypted.key.cache.size\":\"500\",\"hadoop.security.java.secure.random.algorithm\":\"SHA1PRNG\",\"ha.failover-controller.cli-check.rpc-timeout.ms\":\"20000\",\"mapreduce.jobhistory.jobname.limit\":\"50\",\"fs.s3a.select.input.compression\":\"none\",\"yarn.client.nodemanager-connect.retry-interval-ms\":\"10000\",\"ipc.[port_number].scheduler.priority.levels\":\"4\",\"yarn.timeline-service.state-store-class\":\"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore\",\"yarn.nodemanager.env-whitelist\":\"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ\",\"yarn.sharedcache.nested-level\":\"3\",\"yarn.timeline-service.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"fs.azure.user.agent.prefix\":\"unknown\",\"yarn.resourcemanager.zk-delegation-token-node.split-index\":\"*********(redacted)\",\"yarn.nodemanager.numa-awareness.read-topology\":\"false\",\"yarn.nodemanager.webapp.address\":\"${yarn.nodemanager.hostname}:8042\",\"rpc.metrics.quantile.enable\":\"false\",\"yarn.registry.class\":\"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService\",\"mapreduce.jobhistory.admin.acl\":\"*\",\"yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size\":\"10\",\"yarn.scheduler.queue-placement-rules\":\"user-group\",\"hadoop.http.authentication.kerberos.keytab\":\"${user.home}/hadoop.keytab\",\"yarn.resourcemanager.recovery.enabled\":\"false\",\"fs.s3a.select.input.csv.header\":\"none\",\"yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size\":\"500\",\"yarn.timeline-service.webapp.rest-csrf.enabled\":\"false\",\"yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb\":\"0\"},\"System Properties\":{\"java.io.tmpdir\":\"/tmp\",\"line.separator\":\"\\n\",\"path.separator\":\":\",\"sun.management.compiler\":\"HotSpot 64-Bit Tiered Compilers\",\"SPARK_SUBMIT\":\"true\",\"sun.cpu.endian\":\"little\",\"java.specification.maintenance.version\":\"2\",\"java.specification.version\":\"11\",\"java.vm.specification.name\":\"Java Virtual Machine Specification\",\"java.vendor\":\"Ubuntu\",\"java.vm.specification.version\":\"11\",\"user.home\":\"/root\",\"sun.arch.data.model\":\"64\",\"sun.boot.library.path\":\"/usr/lib/jvm/java-11-openjdk-amd64/lib\",\"user.dir\":\"/content\",\"java.library.path\":\"/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib\",\"sun.cpu.isalist\":\"\",\"os.arch\":\"amd64\",\"java.vm.version\":\"11.0.24+8-post-Ubuntu-1ubuntu322.04\",\"jetty.git.hash\":\"cef3fbd6d736a21e7d541a5db490381d95a2047d\",\"java.runtime.version\":\"11.0.24+8-post-Ubuntu-1ubuntu322.04\",\"java.vm.info\":\"mixed mode, sharing\",\"java.runtime.name\":\"OpenJDK Runtime Environment\",\"java.version.date\":\"2024-07-16\",\"file.separator\":\"/\",\"java.class.version\":\"55.0\",\"java.specification.name\":\"Java Platform API Specification\",\"file.encoding\":\"UTF-8\",\"jdk.reflect.useDirectMethodHandle\":\"false\",\"user.timezone\":\"Etc/UTC\",\"java.specification.vendor\":\"Oracle Corporation\",\"sun.java.launcher\":\"SUN_STANDARD\",\"java.vm.compressedOopsMode\":\"32-bit\",\"os.version\":\"6.1.85+\",\"sun.os.patch.level\":\"unknown\",\"java.vm.specification.vendor\":\"Oracle Corporation\",\"user.country\":\"US\",\"sun.jnu.encoding\":\"UTF-8\",\"user.language\":\"en\",\"java.vendor.url\":\"https://ubuntu.com/\",\"java.awt.printerjob\":\"sun.print.PSPrinterJob\",\"java.awt.graphicsenv\":\"sun.awt.X11GraphicsEnvironment\",\"awt.toolkit\":\"sun.awt.X11.XToolkit\",\"os.name\":\"Linux\",\"java.vm.vendor\":\"Ubuntu\",\"jdk.debug\":\"release\",\"java.vendor.url.bug\":\"https://bugs.launchpad.net/ubuntu/+source/openjdk-lts\",\"user.name\":\"root\",\"java.vm.name\":\"OpenJDK 64-Bit Server VM\",\"sun.java.command\":\"org.apache.spark.deploy.SparkSubmit --master spark://651dec53631c:7077 --conf spark.eventLog.enabled=true --class org.apache.spark.examples.SparkPi /content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar 100\",\"java.home\":\"/usr/lib/jvm/java-11-openjdk-amd64\",\"java.version\":\"11.0.24\",\"sun.io.unicode.encoding\":\"UnicodeLittle\"},\"Metrics Properties\":{\"*.sink.servlet.class\":\"org.apache.spark.metrics.sink.MetricsServlet\",\"*.sink.servlet.path\":\"/metrics/json\",\"applications.sink.servlet.path\":\"/metrics/applications/json\",\"master.sink.servlet.path\":\"/metrics/master/json\"},\"Classpath Entries\":{\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-resource-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/chill_2.12-0.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-serde-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/paranamer-2.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-collections4-4.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-dataformat-yaml-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-cli-1.5.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/joda-time-2.12.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-jvm-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-repl_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-math3-3.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-all-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-graphx_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-exec-2.3.9-core.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arpack_combined_all-0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jcl-over-slf4j-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kryo-shaded-4.0.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arpack-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/rocksdbjni-8.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-extensions-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zstd-jni-1.5.5-4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/lapack-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-certificates-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/okio-1.17.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-client-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/ST4-4.0.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sql_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-metrics-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/pickle-1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jline-2.14.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/leveldbjni-all-1.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-core-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-unsafe_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zjsonpatch-0.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-netty-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/guava-14.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-launcher_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-locator-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/xbean-asm9-shaded-4.23.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-lang-2.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-logging-1.1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/shims-0.9.45.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datasketches-java-3.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-core-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/libfb303-0.9.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/annotations-17.0.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-kvstore_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-batch-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-slf4j2-impl-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/libthrift-0.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-metastore-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-framework-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-networking-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-core_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-codec-1.16.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/tink-1.9.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-1.2-api-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-compress-1.23.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-json-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-compiler-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-coordination-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-common-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/derby-10.14.2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/super-csv-2.2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-io-2.16.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/conf/\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/transaction-api-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-text-1.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/xz-1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/ivy-2.5.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.inject-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jdo-api-3.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-core-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/blas-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/httpclient-4.5.14.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datasketches-memory-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-engine-2.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-common-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/aircompressor-0.27.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-events-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-tags_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snappy-java-1.1.10.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/opencsv-2.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-core-1.9.4-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-beeline-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/audience-annotations-0.5.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-jmx-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sql-api_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-compiler-3.1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jpam-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/stream-2.9.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mesos_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-common-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-common-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/JLargeArrays-1.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-scheduling-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-resolver-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javassist-3.29.2-GA.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-mapred-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/minlog-1.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jta-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/chill-java-0.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-node-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-format-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/compress-lzf-1.1.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/objenesis-3.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-server-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-shims-1.9.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-cli-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-encoding-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-api-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/activation-1.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-column-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-api-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json-1.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jsr305-3.0.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/JTransforms-3.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/stax-api-1.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-proxy-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-databind-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-graphite-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/gson-2.2.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jul-to-slf4j-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-policy-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/antlr-runtime-3.5.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/breeze_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-lang3-3.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-network-common_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-common-utils_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-parser-combinators_2.12-2.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-llap-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-jackson-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-reflect-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-dbcp-1.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/logging-interceptor-3.12.12.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-core-4.1.17.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-buffer-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/univocity-parsers-2.9.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-api-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-hk2-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sketch_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-hive_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-storage-api-2.8.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-annotations-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-storageclass-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-kubernetes_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http2-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/httpcore-4.4.16.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/oro-2.0.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-client-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-utils-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-format-structures-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/okhttp-3.12.12.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-core-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/threeten-extra-1.7.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-yarn_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-crypto-1.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-recipes-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-jdbc-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-autoscaling-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/RoaringBitmap-0.9.45.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/HikariCP-2.5.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-vector-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-pool-1.5.4.jar\":\"System Classpath\",\"spark://651dec53631c:37939/jars/spark-examples_2.12-3.5.3.jar\":\"Added By User\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-catalyst_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jodd-core-3.5.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-discovery-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-collections-3.2.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-3.6.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javolution-5.5.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/slf4j-api-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/py4j-0.10.9.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib-local_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/lz4-java-1.8.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-module-scala_2.12-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-network-shuffle_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-socks-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-ipc-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-streaming_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-core-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/algebra_2.12-2.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apiextensions-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apps-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/janino-3.1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-library-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-rbac-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-hadoop-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-datatype-jsr310-2.15.2.jar\":\"System Classpath\"}}\n",
"{\"Event\":\"SparkListenerApplicationStart\",\"App Name\":\"Spark Pi\",\"App ID\":\"app-20241007115024-0000\",\"Timestamp\":1728301820670,\"User\":\"root\"}\n",
"{\"Event\":\"SparkListenerJobStart\",\"Job ID\":0,\"Submission Time\":1728301829820,\"Stage Infos\":[{\"Stage ID\":0,\"Stage Attempt ID\":0,\"Stage Name\":\"reduce at SparkPi.scala:38\",\"Number of Tasks\":100,\"RDD Info\":[{\"RDD ID\":1,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"map\\\"}\",\"Callsite\":\"map at SparkPi.scala:34\",\"Parent IDs\":[0],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":100,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":0,\"Name\":\"ParallelCollectionRDD\",\"Scope\":\"{\\\"id\\\":\\\"0\\\",\\\"name\\\":\\\"parallelize\\\"}\",\"Callsite\":\"parallelize at SparkPi.scala:34\",\"Parent IDs\":[],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":100,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[],\"Details\":\"org.apache.spark.rdd.RDD.reduce(RDD.scala:1121)\\norg.apache.spark.examples.SparkPi$.main(SparkPi.scala:38)\\norg.apache.spark.examples.SparkPi.main(SparkPi.scala)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0}],\"Stage IDs\":[0],\"Properties\":{\"spark.rdd.scope\":\"{\\\"id\\\":\\\"2\\\",\\\"name\\\":\\\"reduce\\\"}\",\"spark.rdd.scope.noOverride\":\"true\"}}\n",
"{\"Event\":\"SparkListenerStageSubmitted\",\"Stage Info\":{\"Stage ID\":0,\"Stage Attempt ID\":0,\"Stage Name\":\"reduce at SparkPi.scala:38\",\"Number of Tasks\":100,\"RDD Info\":[{\"RDD ID\":1,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"map\\\"}\",\"Callsite\":\"map at SparkPi.scala:34\",\"Parent IDs\":[0],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":100,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":0,\"Name\":\"ParallelCollectionRDD\",\"Scope\":\"{\\\"id\\\":\\\"0\\\",\\\"name\\\":\\\"parallelize\\\"}\",\"Callsite\":\"parallelize at SparkPi.scala:34\",\"Parent IDs\":[],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":100,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[],\"Details\":\"org.apache.spark.rdd.RDD.reduce(RDD.scala:1121)\\norg.apache.spark.examples.SparkPi$.main(SparkPi.scala:38)\\norg.apache.spark.examples.SparkPi.main(SparkPi.scala)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Submission Time\":1728301830104,\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0},\"Properties\":{\"spark.rdd.scope\":\"{\\\"id\\\":\\\"2\\\",\\\"name\\\":\\\"reduce\\\"}\",\"spark.rdd.scope.noOverride\":\"true\"}}\n",
"{\"Event\":\"SparkListenerExecutorAdded\",\"Timestamp\":1728301838229,\"Executor ID\":\"0\",\"Executor Info\":{\"Host\":\"172.28.0.12\",\"Total Cores\":2,\"Log Urls\":{\"stdout\":\"http://172.28.0.12:8082/logPage/?appId=app-20241007115024-0000&executorId=0&logType=stdout\",\"stderr\":\"http://172.28.0.12:8082/logPage/?appId=app-20241007115024-0000&executorId=0&logType=stderr\"},\"Attributes\":{},\"Resources\":{},\"Resource Profile Id\":0,\"Registration Time\":1728301838229}}\n",
"{\"Event\":\"SparkListenerBlockManagerAdded\",\"Block Manager ID\":{\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Port\":39293},\"Maximum Memory\":455501414,\"Timestamp\":1728301838480,\"Maximum Onheap Memory\":455501414,\"Maximum Offheap Memory\":0}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":0,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301838872,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":1,\"Index\":1,\"Attempt\":0,\"Partition ID\":1,\"Launch Time\":1728301838975,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":2,\"Index\":2,\"Attempt\":0,\"Partition ID\":2,\"Launch Time\":1728301840899,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":3,\"Index\":3,\"Attempt\":0,\"Partition ID\":3,\"Launch Time\":1728301840933,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Type\":\"ResultTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":1,\"Index\":1,\"Attempt\":0,\"Partition ID\":1,\"Launch Time\":1728301838975,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301840952,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":0,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":1424,\"Value\":1424,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":1,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":126236942,\"Value\":126236942,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":2,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":290,\"Value\":290,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":3,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":72587498,\"Value\":72587498,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":4,\"Name\":\"internal.metrics.resultSize\",\"Update\":1012,\"Value\":1012,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":16,\"Value\":16,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":1424,\"Executor Deserialize CPU Time\":126236942,\"Executor Run Time\":290,\"Executor CPU Time\":72587498,\"Peak Execution Memory\":0,\"Result Size\":1012,\"JVM GC Time\":0,\"Result Serialization Time\":16,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":0,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":0,\"Total Records Read\":0,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":0,\"Shuffle Write Time\":0,\"Shuffle Records Written\":0},\"Input Metrics\":{\"Bytes Read\":0,\"Records Read\":0},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Type\":\"ResultTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":0,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301838872,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301840971,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":0,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":1431,\"Value\":2855,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":1,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":385065942,\"Value\":511302884,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":2,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":286,\"Value\":576,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":3,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":73833811,\"Value\":146421309,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":4,\"Name\":\"internal.metrics.resultSize\",\"Update\":1012,\"Value\":2024,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":9,\"Value\":25,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":1431,\"Executor Deserialize CPU Time\":385065942,\"Executor Run Time\":286,\"Executor CPU Time\":73833811,\"Peak Execution Memory\":0,\"Result Size\":1012,\"JVM GC Time\":0,\"Result Serialization Time\":9,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":0,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":0,\"Total Records Read\":0,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":0,\"Shuffle Write Time\":0,\"Shuffle Records Written\":0},\"Input Metrics\":{\"Bytes Read\":0,\"Records Read\":0},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":4,\"Index\":4,\"Attempt\":0,\"Partition ID\":4,\"Launch Time\":1728301841176,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Type\":\"ResultTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":2,\"Index\":2,\"Attempt\":0,\"Partition ID\":2,\"Launch Time\":1728301840899,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301841222,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":0,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":60,\"Value\":2915,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":1,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":6552543,\"Value\":517855427,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":2,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":126,\"Value\":702,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":3,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":26924342,\"Value\":173345651,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":4,\"Name\":\"internal.metrics.resultSize\",\"Update\":1012,\"Value\":3036,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":6,\"Value\":31,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":60,\"Executor Deserialize CPU Time\":6552543,\"Executor Run Time\":126,\"Executor CPU Time\":26924342,\"Peak Execution Memory\":0,\"Result Size\":1012,\"JVM GC Time\":0,\"Result Serialization Time\":6,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":0,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":0,\"Total Records Read\":0,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":0,\"Shuffle Write Time\":0,\"Shuffle Records Written\":0},\"Input Metrics\":{\"Bytes Read\":0,\"Records Read\":0},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":5,\"Index\":5,\"Attempt\":0,\"Partition ID\":5,\"Launch Time\":1728301841355,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Type\":\"ResultTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":3,\"Index\":3,\"Attempt\":0,\"Partition ID\":3,\"Launch Time\":1728301840933,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301841356,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":0,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":45,\"Value\":2960,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":1,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":5884696,\"Value\":523740123,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":2,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":281,\"Value\":983,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":3,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":27269942,\"Value\":200615593,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":4,\"Name\":\"internal.metrics.resultSize\",\"Update\":1012,\"Value\":4048,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":17,\"Value\":48,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":45,\"Executor Deserialize CPU Time\":5884696,\"Executor Run Time\":281,\"Executor CPU Time\":27269942,\"Peak Execution Memory\":0,\"Result Size\":1012,\"JVM GC Time\":0,\"Result Serialization Time\":17,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":0,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":0,\"Total Records Read\":0,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":0,\"Shuffle Write Time\":0,\"Shuffle Records Written\":0},\"Input Metrics\":{\"Bytes Read\":0,\"Records Read\":0},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":6,\"Index\":6,\"Attempt\":0,\"Partition ID\":6,\"Launch Time\":1728301841462,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"\n",
"==> /tmp/spark-events/app-20241007115053-0001 <==\n",
"{\"Event\":\"SparkListenerLogStart\",\"Spark Version\":\"3.5.3\"}\n",
"{\"Event\":\"SparkListenerResourceProfileAdded\",\"Resource Profile Id\":0,\"Executor Resource Requests\":{\"memory\":{\"Resource Name\":\"memory\",\"Amount\":1024,\"Discovery Script\":\"\",\"Vendor\":\"\"},\"offHeap\":{\"Resource Name\":\"offHeap\",\"Amount\":0,\"Discovery Script\":\"\",\"Vendor\":\"\"}},\"Task Resource Requests\":{\"cpus\":{\"Resource Name\":\"cpus\",\"Amount\":1.0}}}\n",
"{\"Event\":\"SparkListenerBlockManagerAdded\",\"Block Manager ID\":{\"Executor ID\":\"driver\",\"Host\":\"651dec53631c\",\"Port\":41653},\"Maximum Memory\":455501414,\"Timestamp\":1728301853573,\"Maximum Onheap Memory\":455501414,\"Maximum Offheap Memory\":0}\n",
"{\"Event\":\"SparkListenerEnvironmentUpdate\",\"JVM Information\":{\"Java Home\":\"/usr/lib/jvm/java-11-openjdk-amd64\",\"Java Version\":\"11.0.24 (Ubuntu)\",\"Scala Version\":\"version 2.12.18\"},\"Spark Properties\":{\"spark.executor.extraJavaOptions\":\"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false\",\"spark.driver.host\":\"651dec53631c\",\"spark.eventLog.enabled\":\"true\",\"spark.driver.port\":\"33373\",\"spark.jars\":\"file:/content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar\",\"spark.app.name\":\"JavaWordCount\",\"spark.scheduler.mode\":\"FIFO\",\"spark.submit.pyFiles\":\"\",\"spark.app.submitTime\":\"1728301851272\",\"spark.app.startTime\":\"1728301851395\",\"spark.executor.id\":\"driver\",\"spark.driver.extraJavaOptions\":\"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false\",\"spark.app.initial.jar.urls\":\"spark://651dec53631c:33373/jars/spark-examples_2.12-3.5.3.jar\",\"spark.submit.deployMode\":\"client\",\"spark.master\":\"spark://651dec53631c:7077\",\"spark.app.id\":\"app-20241007115053-0001\"},\"Hadoop Properties\":{\"hadoop.service.shutdown.timeout\":\"30s\",\"yarn.resourcemanager.amlauncher.thread-count\":\"50\",\"yarn.sharedcache.enabled\":\"false\",\"fs.s3a.connection.maximum\":\"96\",\"yarn.nodemanager.numa-awareness.numactl.cmd\":\"/usr/bin/numactl\",\"fs.viewfs.overload.scheme.target.o3fs.impl\":\"org.apache.hadoop.fs.ozone.OzoneFileSystem\",\"fs.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3AFileSystem\",\"yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms\":\"1000\",\"yarn.timeline-service.timeline-client.number-of-async-entities-to-merge\":\"10\",\"hadoop.security.kms.client.timeout\":\"60\",\"hadoop.http.authentication.kerberos.principal\":\"HTTP/_HOST@LOCALHOST\",\"mapreduce.jobhistory.loadedjob.tasks.max\":\"-1\",\"yarn.resourcemanager.application-tag-based-placement.enable\":\"false\",\"mapreduce.framework.name\":\"local\",\"yarn.sharedcache.uploader.server.thread-count\":\"50\",\"yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min\":\"3600\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern\":\"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$\",\"tfile.fs.output.buffer.size\":\"262144\",\"yarn.app.mapreduce.am.job.task.listener.thread-count\":\"30\",\"yarn.nodemanager.node-attributes.resync-interval-ms\":\"120000\",\"yarn.nodemanager.container-log-monitor.interval-ms\":\"60000\",\"hadoop.security.groups.cache.background.reload.threads\":\"3\",\"yarn.resourcemanager.webapp.cross-origin.enabled\":\"false\",\"fs.AbstractFileSystem.ftp.impl\":\"org.apache.hadoop.fs.ftp.FtpFs\",\"fs.viewfs.overload.scheme.target.gs.impl\":\"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS\",\"hadoop.registry.secure\":\"false\",\"hadoop.shell.safely.delete.limit.num.files\":\"100\",\"mapreduce.job.acl-view-job\":\" \",\"fs.s3a.s3guard.ddb.background.sleep\":\"25ms\",\"fs.s3a.retry.limit\":\"7\",\"mapreduce.jobhistory.loadedjobs.cache.size\":\"5\",\"fs.s3a.s3guard.ddb.table.create\":\"false\",\"fs.viewfs.overload.scheme.target.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3AFileSystem\",\"yarn.nodemanager.amrmproxy.enabled\":\"false\",\"yarn.timeline-service.entity-group-fs-store.with-user-dir\":\"false\",\"mapreduce.shuffle.pathcache.expire-after-access-minutes\":\"5\",\"mapreduce.input.fileinputformat.split.minsize\":\"0\",\"yarn.resourcemanager.container.liveness-monitor.interval-ms\":\"600000\",\"yarn.resourcemanager.client.thread-count\":\"50\",\"io.seqfile.compress.blocksize\":\"1000000\",\"yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes\":\"runc\",\"fs.viewfs.overload.scheme.target.http.impl\":\"org.apache.hadoop.fs.http.HttpFileSystem\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor\":\"1.0\",\"yarn.sharedcache.checksum.algo.impl\":\"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl\",\"yarn.nodemanager.amrmproxy.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor\",\"yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size\":\"10485760\",\"mapreduce.reduce.shuffle.fetch.retry.interval-ms\":\"1000\",\"mapreduce.task.profile.maps\":\"0-2\",\"yarn.scheduler.include-port-in-node-name\":\"false\",\"yarn.nodemanager.admin-env\":\"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX\",\"yarn.resourcemanager.node-removal-untracked.timeout-ms\":\"60000\",\"mapreduce.am.max-attempts\":\"2\",\"hadoop.security.kms.client.failover.sleep.base.millis\":\"100\",\"mapreduce.jobhistory.webapp.https.address\":\"0.0.0.0:19890\",\"yarn.node-labels.fs-store.impl.class\":\"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore\",\"yarn.nodemanager.collector-service.address\":\"${yarn.nodemanager.hostname}:8048\",\"fs.trash.checkpoint.interval\":\"0\",\"mapreduce.job.map.output.collector.class\":\"org.apache.hadoop.mapred.MapTask$MapOutputBuffer\",\"yarn.resourcemanager.node-ip-cache.expiry-interval-secs\":\"-1\",\"hadoop.http.authentication.signature.secret.file\":\"*********(redacted)\",\"hadoop.jetty.logs.serve.aliases\":\"true\",\"yarn.resourcemanager.placement-constraints.handler\":\"disabled\",\"yarn.timeline-service.handler-thread-count\":\"10\",\"yarn.resourcemanager.max-completed-applications\":\"1000\",\"yarn.nodemanager.aux-services.manifest.enabled\":\"false\",\"yarn.resourcemanager.system-metrics-publisher.enabled\":\"false\",\"yarn.resourcemanager.placement-constraints.algorithm.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm\",\"yarn.sharedcache.webapp.address\":\"0.0.0.0:8788\",\"fs.s3a.select.input.csv.quote.escape.character\":\"\\\\\\\\\",\"yarn.resourcemanager.delegation.token.renew-interval\":\"*********(redacted)\",\"yarn.sharedcache.nm.uploader.replication.factor\":\"10\",\"hadoop.security.groups.negative-cache.secs\":\"30\",\"yarn.app.mapreduce.task.container.log.backups\":\"0\",\"mapreduce.reduce.skip.proc-count.auto-incr\":\"true\",\"fs.viewfs.overload.scheme.target.swift.impl\":\"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem\",\"hadoop.security.group.mapping.ldap.posix.attr.gid.name\":\"gidNumber\",\"ipc.client.fallback-to-simple-auth-allowed\":\"false\",\"yarn.nodemanager.resource.memory.enforced\":\"true\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch\":\"false\",\"yarn.client.failover-proxy-provider\":\"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider\",\"yarn.timeline-service.http-authentication.simple.anonymous.allowed\":\"true\",\"ha.health-monitor.check-interval.ms\":\"1000\",\"yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed\":\"false\",\"hadoop.metrics.jvm.use-thread-mxbean\":\"false\",\"ipc.[port_number].faircallqueue.multiplexer.weights\":\"8,4,2,1\",\"yarn.acl.reservation-enable\":\"false\",\"yarn.resourcemanager.store.class\":\"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore\",\"yarn.app.mapreduce.am.hard-kill-timeout-ms\":\"10000\",\"fs.s3a.etag.checksum.enabled\":\"false\",\"yarn.nodemanager.container-metrics.enable\":\"true\",\"ha.health-monitor.rpc.connect.max.retries\":\"1\",\"yarn.timeline-service.client.fd-clean-interval-secs\":\"60\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable\":\"false\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-ms\":\"1000\",\"hadoop.common.configuration.version\":\"3.0.0\",\"fs.s3a.s3guard.ddb.table.capacity.read\":\"0\",\"yarn.nodemanager.remote-app-log-dir-suffix\":\"logs\",\"yarn.nodemanager.container-log-monitor.dir-size-limit-bytes\":\"1000000000\",\"yarn.nodemanager.windows-container.cpu-limit.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed\":\"false\",\"file.blocksize\":\"67108864\",\"hadoop.http.idle_timeout.ms\":\"60000\",\"hadoop.registry.zk.retry.ceiling.ms\":\"60000\",\"yarn.scheduler.configuration.leveldb-store.path\":\"${hadoop.tmp.dir}/yarn/system/confstore\",\"yarn.sharedcache.store.in-memory.initial-delay-mins\":\"10\",\"mapreduce.jobhistory.principal\":\"jhs/_HOST@REALM.TLD\",\"mapreduce.map.skip.proc-count.auto-incr\":\"true\",\"fs.s3a.committer.name\":\"file\",\"mapreduce.task.profile.reduces\":\"0-2\",\"hadoop.zk.num-retries\":\"1000\",\"yarn.webapp.xfs-filter.enabled\":\"true\",\"fs.viewfs.overload.scheme.target.hdfs.impl\":\"org.apache.hadoop.hdfs.DistributedFileSystem\",\"seq.io.sort.mb\":\"100\",\"yarn.scheduler.configuration.max.version\":\"100\",\"yarn.timeline-service.webapp.https.address\":\"${yarn.timeline-service.hostname}:8190\",\"yarn.resourcemanager.scheduler.address\":\"${yarn.resourcemanager.hostname}:8030\",\"yarn.node-labels.enabled\":\"false\",\"yarn.resourcemanager.webapp.ui-actions.enabled\":\"true\",\"mapreduce.task.timeout\":\"600000\",\"yarn.sharedcache.client-server.thread-count\":\"50\",\"hadoop.security.groups.shell.command.timeout\":\"0s\",\"hadoop.security.crypto.cipher.suite\":\"AES/CTR/NoPadding\",\"yarn.nodemanager.elastic-memory-control.oom-handler\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler\",\"yarn.resourcemanager.connect.max-wait.ms\":\"900000\",\"fs.defaultFS\":\"file:///\",\"yarn.minicluster.use-rpc\":\"false\",\"ipc.[port_number].decay-scheduler.decay-factor\":\"0.5\",\"fs.har.impl.disable.cache\":\"true\",\"yarn.webapp.ui2.enable\":\"false\",\"io.compression.codec.bzip2.library\":\"system-native\",\"yarn.webapp.filter-invalid-xml-chars\":\"false\",\"yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs\":\"600\",\"fs.s3a.select.input.csv.record.delimiter\":\"\\\\n\",\"fs.s3a.change.detection.source\":\"etag\",\"ipc.[port_number].backoff.enable\":\"false\",\"yarn.nodemanager.distributed-scheduling.enabled\":\"false\",\"mapreduce.shuffle.connection-keep-alive.timeout\":\"5\",\"yarn.resourcemanager.webapp.https.address\":\"${yarn.resourcemanager.hostname}:8090\",\"yarn.webapp.enable-rest-app-submissions\":\"true\",\"mapreduce.jobhistory.address\":\"0.0.0.0:10020\",\"yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"yarn.is.minicluster\":\"false\",\"yarn.nodemanager.address\":\"${yarn.nodemanager.hostname}:0\",\"fs.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem\",\"fs.AbstractFileSystem.s3a.impl\":\"org.apache.hadoop.fs.s3a.S3A\",\"mapreduce.task.combine.progress.records\":\"10000\",\"yarn.resourcemanager.epoch.range\":\"0\",\"yarn.resourcemanager.am.max-attempts\":\"2\",\"yarn.nodemanager.runtime.linux.runc.image-toplevel-dir\":\"/runc-root\",\"yarn.nodemanager.linux-container-executor.cgroups.hierarchy\":\"/hadoop-yarn\",\"fs.AbstractFileSystem.wasbs.impl\":\"org.apache.hadoop.fs.azure.Wasbs\",\"yarn.timeline-service.entity-group-fs-store.cache-store-class\":\"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore\",\"yarn.nodemanager.runtime.linux.runc.allowed-container-networks\":\"host,none,bridge\",\"fs.ftp.transfer.mode\":\"BLOCK_TRANSFER_MODE\",\"ipc.server.log.slow.rpc\":\"false\",\"ipc.server.reuseaddr\":\"true\",\"fs.ftp.timeout\":\"0\",\"yarn.resourcemanager.node-labels.provider.fetch-interval-ms\":\"1800000\",\"yarn.router.webapp.https.address\":\"0.0.0.0:8091\",\"yarn.nodemanager.webapp.cross-origin.enabled\":\"false\",\"fs.wasb.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem\",\"yarn.resourcemanager.auto-update.containers\":\"false\",\"yarn.app.mapreduce.am.job.committer.cancel-timeout\":\"60000\",\"yarn.scheduler.configuration.zk-store.parent-path\":\"/confstore\",\"yarn.nodemanager.default-container-executor.log-dirs.permissions\":\"710\",\"yarn.app.attempt.diagnostics.limit.kc\":\"64\",\"fs.viewfs.overload.scheme.target.swebhdfs.impl\":\"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem\",\"yarn.client.failover-no-ha-proxy-provider\":\"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider\",\"fs.s3a.change.detection.mode\":\"server\",\"ftp.bytes-per-checksum\":\"512\",\"yarn.nodemanager.resource.memory-mb\":\"-1\",\"fs.AbstractFileSystem.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.Abfs\",\"yarn.timeline-service.writer.flush-interval-seconds\":\"60\",\"fs.s3a.fast.upload.active.blocks\":\"4\",\"yarn.resourcemanager.submission-preprocessor.enabled\":\"false\",\"hadoop.security.credential.clear-text-fallback\":\"true\",\"yarn.nodemanager.collector-service.thread-count\":\"5\",\"ipc.[port_number].scheduler.impl\":\"org.apache.hadoop.ipc.DefaultRpcScheduler\",\"fs.azure.secure.mode\":\"false\",\"mapreduce.jobhistory.joblist.cache.size\":\"20000\",\"fs.ftp.host\":\"0.0.0.0\",\"yarn.timeline-service.writer.async.queue.capacity\":\"100\",\"yarn.resourcemanager.fs.state-store.num-retries\":\"0\",\"yarn.resourcemanager.nodemanager-connect-retries\":\"10\",\"yarn.nodemanager.log-aggregation.num-log-files-per-app\":\"30\",\"hadoop.security.kms.client.encrypted.key.cache.low-watermark\":\"0.3f\",\"fs.s3a.committer.magic.enabled\":\"true\",\"yarn.timeline-service.client.max-retries\":\"30\",\"dfs.ha.fencing.ssh.connect-timeout\":\"30000\",\"yarn.log-aggregation-enable\":\"false\",\"yarn.system-metrics-publisher.enabled\":\"false\",\"mapreduce.reduce.markreset.buffer.percent\":\"0.0\",\"fs.AbstractFileSystem.viewfs.impl\":\"org.apache.hadoop.fs.viewfs.ViewFs\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor\":\"1.0\",\"mapreduce.task.io.sort.factor\":\"10\",\"yarn.nodemanager.amrmproxy.client.thread-count\":\"25\",\"ha.failover-controller.new-active.rpc-timeout.ms\":\"60000\",\"yarn.nodemanager.container-localizer.java.opts\":\"-Xmx256m\",\"mapreduce.jobhistory.datestring.cache.size\":\"200000\",\"mapreduce.job.acl-modify-job\":\" \",\"yarn.nodemanager.windows-container.memory-limit.enabled\":\"false\",\"yarn.timeline-service.webapp.address\":\"${yarn.timeline-service.hostname}:8188\",\"yarn.app.mapreduce.am.job.committer.commit-window\":\"10000\",\"yarn.nodemanager.container-manager.thread-count\":\"20\",\"yarn.minicluster.fixed.ports\":\"false\",\"hadoop.tags.system\":\"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL\",\"yarn.cluster.max-application-priority\":\"0\",\"yarn.timeline-service.ttl-enable\":\"true\",\"mapreduce.jobhistory.recovery.store.fs.uri\":\"${hadoop.tmp.dir}/mapred/history/recoverystore\",\"hadoop.caller.context.signature.max.size\":\"40\",\"ipc.[port_number].decay-scheduler.backoff.responsetime.enable\":\"false\",\"yarn.client.load.resource-types.from-server\":\"false\",\"ha.zookeeper.session-timeout.ms\":\"10000\",\"ipc.[port_number].decay-scheduler.metrics.top.user.count\":\"10\",\"tfile.io.chunk.size\":\"1048576\",\"fs.s3a.s3guard.ddb.table.capacity.write\":\"0\",\"yarn.dispatcher.print-events-info.threshold\":\"5000\",\"mapreduce.job.speculative.slowtaskthreshold\":\"1.0\",\"io.serializations\":\"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization\",\"hadoop.security.kms.client.failover.sleep.max.millis\":\"2000\",\"hadoop.security.group.mapping.ldap.directory.search.timeout\":\"10000\",\"yarn.scheduler.configuration.store.max-logs\":\"1000\",\"yarn.nodemanager.node-attributes.provider.fetch-interval-ms\":\"600000\",\"fs.swift.impl\":\"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem\",\"yarn.nodemanager.local-cache.max-files-per-directory\":\"8192\",\"hadoop.http.cross-origin.enabled\":\"false\",\"hadoop.zk.acl\":\"world:anyone:rwcda\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache\":\"10\",\"mapreduce.map.sort.spill.percent\":\"0.80\",\"yarn.timeline-service.entity-group-fs-store.scan-interval-seconds\":\"60\",\"yarn.node-attribute.fs-store.impl.class\":\"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore\",\"fs.s3a.retry.interval\":\"500ms\",\"yarn.timeline-service.client.best-effort\":\"false\",\"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\":\"*********(redacted)\",\"hadoop.security.group.mapping.ldap.posix.attr.uid.name\":\"uidNumber\",\"fs.AbstractFileSystem.swebhdfs.impl\":\"org.apache.hadoop.fs.SWebHdfs\",\"yarn.nodemanager.elastic-memory-control.timeout-sec\":\"5\",\"fs.s3a.select.enabled\":\"true\",\"mapreduce.ifile.readahead\":\"true\",\"yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms\":\"300000\",\"yarn.timeline-service.reader.webapp.address\":\"${yarn.timeline-service.webapp.address}\",\"yarn.resourcemanager.placement-constraints.algorithm.pool-size\":\"1\",\"yarn.timeline-service.hbase.coprocessor.jar.hdfs.location\":\"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar\",\"hadoop.security.kms.client.encrypted.key.cache.num.refill.threads\":\"2\",\"yarn.resourcemanager.scheduler.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler\",\"yarn.app.mapreduce.am.command-opts\":\"-Xmx1024m\",\"fs.s3a.metadatastore.fail.on.write.error\":\"true\",\"hadoop.http.sni.host.check.enabled\":\"false\",\"mapreduce.cluster.local.dir\":\"${hadoop.tmp.dir}/mapred/local\",\"io.mapfile.bloom.error.rate\":\"0.005\",\"fs.client.resolve.topology.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.allowed-runtimes\":\"default\",\"yarn.sharedcache.store.class\":\"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore\",\"ha.failover-controller.graceful-fence.rpc-timeout.ms\":\"5000\",\"ftp.replication\":\"3\",\"fs.getspaceused.jitterMillis\":\"60000\",\"hadoop.security.uid.cache.secs\":\"14400\",\"mapreduce.job.maxtaskfailures.per.tracker\":\"3\",\"fs.s3a.metadatastore.impl\":\"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore\",\"io.skip.checksum.errors\":\"false\",\"yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts\":\"3\",\"yarn.timeline-service.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"fs.s3a.connection.timeout\":\"200000\",\"yarn.app.mapreduce.am.webapp.https.enabled\":\"false\",\"mapreduce.job.max.split.locations\":\"15\",\"yarn.resourcemanager.nm-container-queuing.max-queue-length\":\"15\",\"yarn.resourcemanager.delegation-token.always-cancel\":\"*********(redacted)\",\"hadoop.registry.zk.session.timeout.ms\":\"60000\",\"yarn.federation.cache-ttl.secs\":\"300\",\"mapreduce.jvm.system-properties-to-log\":\"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name\",\"yarn.resourcemanager.opportunistic-container-allocation.nodes-used\":\"10\",\"yarn.timeline-service.entity-group-fs-store.active-dir\":\"/tmp/entity-file-history/active\",\"mapreduce.shuffle.transfer.buffer.size\":\"131072\",\"yarn.timeline-service.client.retry-interval-ms\":\"1000\",\"yarn.timeline-service.flowname.max-size\":\"0\",\"yarn.http.policy\":\"HTTP_ONLY\",\"fs.s3a.socket.send.buffer\":\"8192\",\"fs.AbstractFileSystem.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.Abfss\",\"yarn.sharedcache.uploader.server.address\":\"0.0.0.0:8046\",\"yarn.resourcemanager.delegation-token.max-conf-size-bytes\":\"*********(redacted)\",\"hadoop.http.authentication.token.validity\":\"*********(redacted)\",\"mapreduce.shuffle.max.connections\":\"0\",\"yarn.minicluster.yarn.nodemanager.resource.memory-mb\":\"4096\",\"mapreduce.job.emit-timeline-data\":\"false\",\"yarn.nodemanager.resource.system-reserved-memory-mb\":\"-1\",\"hadoop.kerberos.min.seconds.before.relogin\":\"60\",\"mapreduce.jobhistory.move.thread-count\":\"3\",\"yarn.resourcemanager.admin.client.thread-count\":\"1\",\"yarn.dispatcher.drain-events.timeout\":\"300000\",\"ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds\":\"10s,20s,30s,40s\",\"fs.s3a.buffer.dir\":\"${hadoop.tmp.dir}/s3a\",\"hadoop.ssl.enabled.protocols\":\"TLSv1.2\",\"mapreduce.jobhistory.admin.address\":\"0.0.0.0:10033\",\"yarn.log-aggregation-status.time-out.ms\":\"600000\",\"fs.s3a.accesspoint.required\":\"false\",\"mapreduce.shuffle.port\":\"13562\",\"yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory\":\"10\",\"yarn.nodemanager.health-checker.interval-ms\":\"600000\",\"yarn.resourcemanager.proxy.connection.timeout\":\"60000\",\"yarn.router.clientrm.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor\",\"yarn.resourcemanager.zk-appid-node.split-index\":\"0\",\"ftp.blocksize\":\"67108864\",\"yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions\":\"read\",\"yarn.router.rmadmin.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor\",\"yarn.nodemanager.log-container-debug-info.enabled\":\"true\",\"yarn.resourcemanager.activities-manager.app-activities.max-queue-length\":\"100\",\"yarn.resourcemanager.application-https.policy\":\"NONE\",\"yarn.client.max-cached-nodemanagers-proxies\":\"0\",\"yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms\":\"20\",\"yarn.nodemanager.delete.debug-delay-sec\":\"0\",\"yarn.nodemanager.pmem-check-enabled\":\"true\",\"yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage\":\"90.0\",\"mapreduce.app-submission.cross-platform\":\"false\",\"yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms\":\"10000\",\"yarn.nodemanager.container-retry-minimum-interval-ms\":\"1000\",\"hadoop.security.groups.cache.secs\":\"300\",\"yarn.federation.enabled\":\"false\",\"yarn.workflow-id.tag-prefix\":\"workflowid:\",\"fs.azure.local.sas.key.mode\":\"false\",\"ipc.maximum.data.length\":\"134217728\",\"fs.s3a.endpoint\":\"s3.amazonaws.com\",\"mapreduce.shuffle.max.threads\":\"0\",\"yarn.router.pipeline.cache-max-size\":\"25\",\"yarn.resourcemanager.nm-container-queuing.load-comparator\":\"QUEUE_LENGTH\",\"yarn.resourcemanager.resource-tracker.nm.ip-hostname-check\":\"false\",\"hadoop.security.authorization\":\"false\",\"mapreduce.job.complete.cancel.delegation.tokens\":\"*********(redacted)\",\"fs.s3a.paging.maximum\":\"5000\",\"nfs.exports.allowed.hosts\":\"* rw\",\"yarn.nodemanager.amrmproxy.ha.enable\":\"false\",\"fs.AbstractFileSystem.gs.impl\":\"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS\",\"mapreduce.jobhistory.http.policy\":\"HTTP_ONLY\",\"yarn.sharedcache.store.in-memory.check-period-mins\":\"720\",\"hadoop.security.group.mapping.ldap.ssl\":\"false\",\"fs.s3a.downgrade.syncable.exceptions\":\"true\",\"yarn.client.application-client-protocol.poll-interval-ms\":\"200\",\"yarn.scheduler.configuration.leveldb-store.compaction-interval-secs\":\"86400\",\"yarn.timeline-service.writer.class\":\"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl\",\"ha.zookeeper.parent-znode\":\"/hadoop-ha\",\"yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms\":\"60000\",\"yarn.nodemanager.log-aggregation.policy.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy\",\"mapreduce.reduce.shuffle.merge.percent\":\"0.66\",\"hadoop.security.group.mapping.ldap.search.filter.group\":\"(objectClass=group)\",\"yarn.resourcemanager.placement-constraints.scheduler.pool-size\":\"1\",\"yarn.resourcemanager.activities-manager.cleanup-interval-ms\":\"5000\",\"yarn.nodemanager.resourcemanager.minimum.version\":\"NONE\",\"mapreduce.job.speculative.speculative-cap-running-tasks\":\"0.1\",\"yarn.admin.acl\":\"*\",\"ipc.[port_number].identity-provider.impl\":\"org.apache.hadoop.ipc.UserIdentityProvider\",\"yarn.nodemanager.recovery.supervised\":\"false\",\"yarn.sharedcache.admin.thread-count\":\"1\",\"yarn.resourcemanager.ha.automatic-failover.enabled\":\"true\",\"yarn.nodemanager.container-log-monitor.total-size-limit-bytes\":\"10000000000\",\"mapreduce.reduce.skip.maxgroups\":\"0\",\"mapreduce.reduce.shuffle.connect.timeout\":\"180000\",\"yarn.nodemanager.health-checker.scripts\":\"script\",\"yarn.resourcemanager.address\":\"${yarn.resourcemanager.hostname}:8032\",\"ipc.client.ping\":\"true\",\"mapreduce.task.local-fs.write-limit.bytes\":\"-1\",\"fs.adl.oauth2.access.token.provider.type\":\"*********(redacted)\",\"mapreduce.shuffle.ssl.file.buffer.size\":\"65536\",\"yarn.resourcemanager.ha.automatic-failover.embedded\":\"true\",\"yarn.nodemanager.resource-plugins.gpu.docker-plugin\":\"nvidia-docker-v1\",\"fs.s3a.s3guard.consistency.retry.interval\":\"2s\",\"fs.s3a.multipart.purge\":\"false\",\"yarn.scheduler.configuration.store.class\":\"file\",\"yarn.resourcemanager.nm-container-queuing.queue-limit-stdev\":\"1.0f\",\"mapreduce.job.end-notification.max.attempts\":\"5\",\"mapreduce.output.fileoutputformat.compress.codec\":\"org.apache.hadoop.io.compress.DefaultCodec\",\"yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled\":\"false\",\"ipc.client.bind.wildcard.addr\":\"false\",\"yarn.resourcemanager.webapp.rest-csrf.enabled\":\"false\",\"ha.health-monitor.connect-retry-interval.ms\":\"1000\",\"yarn.nodemanager.keytab\":\"/etc/krb5.keytab\",\"mapreduce.jobhistory.keytab\":\"/etc/security/keytab/jhs.service.keytab\",\"fs.s3a.threads.max\":\"64\",\"yarn.nodemanager.runtime.linux.docker.image-update\":\"false\",\"mapreduce.reduce.shuffle.input.buffer.percent\":\"0.70\",\"fs.viewfs.overload.scheme.target.abfss.impl\":\"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem\",\"yarn.dispatcher.cpu-monitor.samples-per-min\":\"60\",\"hadoop.security.token.service.use_ip\":\"*********(redacted)\",\"yarn.nodemanager.runtime.linux.docker.allowed-container-networks\":\"host,none,bridge\",\"yarn.nodemanager.node-labels.resync-interval-ms\":\"120000\",\"hadoop.tmp.dir\":\"/tmp/hadoop-${user.name}\",\"mapreduce.job.maps\":\"2\",\"mapreduce.jobhistory.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.job.end-notification.max.retry.interval\":\"5000\",\"yarn.log-aggregation.retain-check-interval-seconds\":\"-1\",\"yarn.resourcemanager.resource-tracker.client.thread-count\":\"50\",\"yarn.nodemanager.containers-launcher.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher\",\"yarn.rm.system-metrics-publisher.emit-container-events\":\"false\",\"yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size\":\"10000\",\"yarn.resourcemanager.ha.automatic-failover.zk-base-path\":\"/yarn-leader-election\",\"io.seqfile.local.dir\":\"${hadoop.tmp.dir}/io/local\",\"fs.s3a.s3guard.ddb.throttle.retry.interval\":\"100ms\",\"fs.AbstractFileSystem.wasb.impl\":\"org.apache.hadoop.fs.azure.Wasb\",\"mapreduce.client.submit.file.replication\":\"10\",\"mapreduce.jobhistory.minicluster.fixed.ports\":\"false\",\"fs.s3a.multipart.threshold\":\"128M\",\"yarn.resourcemanager.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"mapreduce.jobhistory.done-dir\":\"${yarn.app.mapreduce.am.staging-dir}/history/done\",\"ipc.server.purge.interval\":\"15\",\"ipc.client.idlethreshold\":\"4000\",\"yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage\":\"false\",\"mapreduce.reduce.input.buffer.percent\":\"0.0\",\"yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold\":\"1\",\"yarn.nodemanager.webapp.rest-csrf.enabled\":\"false\",\"fs.ftp.host.port\":\"21\",\"ipc.ping.interval\":\"60000\",\"yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size\":\"10\",\"yarn.resourcemanager.admin.address\":\"${yarn.resourcemanager.hostname}:8033\",\"file.client-write-packet-size\":\"65536\",\"ipc.client.kill.max\":\"10\",\"mapreduce.reduce.speculative\":\"true\",\"hadoop.security.key.default.bitlength\":\"128\",\"mapreduce.job.reducer.unconditional-preempt.delay.sec\":\"300\",\"yarn.nodemanager.disk-health-checker.interval-ms\":\"120000\",\"yarn.nodemanager.log.deletion-threads-count\":\"4\",\"fs.s3a.committer.abort.pending.uploads\":\"true\",\"yarn.webapp.filter-entity-list-by-user\":\"false\",\"yarn.resourcemanager.activities-manager.app-activities.ttl-ms\":\"600000\",\"ipc.client.connection.maxidletime\":\"10000\",\"mapreduce.task.io.sort.mb\":\"100\",\"yarn.nodemanager.localizer.client.thread-count\":\"5\",\"io.erasurecode.codec.rs.rawcoders\":\"rs_native,rs_java\",\"io.erasurecode.codec.rs-legacy.rawcoders\":\"rs-legacy_java\",\"yarn.sharedcache.admin.address\":\"0.0.0.0:8047\",\"yarn.resourcemanager.placement-constraints.algorithm.iterator\":\"SERIAL\",\"yarn.nodemanager.localizer.cache.cleanup.interval-ms\":\"600000\",\"hadoop.security.crypto.codec.classes.aes.ctr.nopadding\":\"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec\",\"mapreduce.job.cache.limit.max-resources-mb\":\"0\",\"fs.s3a.connection.ssl.enabled\":\"true\",\"yarn.nodemanager.process-kill-wait.ms\":\"5000\",\"mapreduce.job.hdfs-servers\":\"${fs.defaultFS}\",\"yarn.app.mapreduce.am.webapp.https.client.auth\":\"false\",\"hadoop.workaround.non.threadsafe.getpwuid\":\"true\",\"fs.df.interval\":\"60000\",\"ipc.[port_number].decay-scheduler.thresholds\":\"13,25,50\",\"fs.s3a.multiobjectdelete.enable\":\"true\",\"yarn.sharedcache.cleaner.resource-sleep-ms\":\"0\",\"yarn.nodemanager.disk-health-checker.min-healthy-disks\":\"0.25\",\"hadoop.shell.missing.defaultFs.warning\":\"false\",\"io.file.buffer.size\":\"65536\",\"fs.viewfs.overload.scheme.target.wasb.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem\",\"hadoop.security.group.mapping.ldap.search.attr.member\":\"member\",\"hadoop.security.random.device.file.path\":\"/dev/urandom\",\"hadoop.security.sensitive-config-keys\":\"*********(redacted)\",\"fs.s3a.s3guard.ddb.max.retries\":\"9\",\"fs.viewfs.overload.scheme.target.file.impl\":\"org.apache.hadoop.fs.LocalFileSystem\",\"hadoop.rpc.socket.factory.class.default\":\"org.apache.hadoop.net.StandardSocketFactory\",\"yarn.intermediate-data-encryption.enable\":\"false\",\"yarn.resourcemanager.connect.retry-interval.ms\":\"30000\",\"yarn.nodemanager.container.stderr.pattern\":\"{*stderr*,*STDERR*}\",\"yarn.scheduler.minimum-allocation-mb\":\"1024\",\"yarn.app.mapreduce.am.staging-dir\":\"/tmp/hadoop-yarn/staging\",\"mapreduce.reduce.shuffle.read.timeout\":\"180000\",\"hadoop.http.cross-origin.max-age\":\"1800\",\"io.erasurecode.codec.xor.rawcoders\":\"xor_native,xor_java\",\"fs.s3a.s3guard.consistency.retry.limit\":\"7\",\"fs.s3a.connection.establish.timeout\":\"5000\",\"mapreduce.job.running.map.limit\":\"0\",\"yarn.minicluster.control-resource-monitoring\":\"false\",\"hadoop.ssl.require.client.cert\":\"false\",\"hadoop.kerberos.kinit.command\":\"kinit\",\"yarn.federation.state-store.class\":\"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore\",\"mapreduce.reduce.log.level\":\"INFO\",\"hadoop.security.dns.log-slow-lookups.threshold.ms\":\"1000\",\"mapreduce.job.ubertask.enable\":\"false\",\"adl.http.timeout\":\"-1\",\"yarn.resourcemanager.placement-constraints.retry-attempts\":\"3\",\"hadoop.caller.context.enabled\":\"false\",\"hadoop.security.group.mapping.ldap.num.attempts\":\"3\",\"yarn.nodemanager.vmem-pmem-ratio\":\"2.1\",\"hadoop.rpc.protection\":\"authentication\",\"ha.health-monitor.rpc-timeout.ms\":\"45000\",\"yarn.nodemanager.remote-app-log-dir\":\"/tmp/logs\",\"hadoop.zk.timeout-ms\":\"10000\",\"fs.s3a.s3guard.cli.prune.age\":\"86400000\",\"yarn.nodemanager.resource.pcores-vcores-multiplier\":\"1.0\",\"yarn.nodemanager.runtime.linux.sandbox-mode\":\"disabled\",\"yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size\":\"10\",\"fs.viewfs.overload.scheme.target.webhdfs.impl\":\"org.apache.hadoop.hdfs.web.WebHdfsFileSystem\",\"fs.s3a.committer.threads\":\"8\",\"hadoop.zk.retry-interval-ms\":\"1000\",\"hadoop.security.crypto.buffer.size\":\"8192\",\"yarn.nodemanager.node-labels.provider.fetch-interval-ms\":\"600000\",\"mapreduce.jobhistory.recovery.store.leveldb.path\":\"${hadoop.tmp.dir}/mapred/history/recoverystore\",\"yarn.client.failover-retries-on-socket-timeouts\":\"0\",\"fs.s3a.ssl.channel.mode\":\"default_jsse\",\"yarn.nodemanager.resource.memory.enabled\":\"false\",\"fs.azure.authorization.caching.enable\":\"true\",\"hadoop.security.instrumentation.requires.admin\":\"false\",\"yarn.nodemanager.delete.thread-count\":\"4\",\"mapreduce.job.finish-when-all-reducers-done\":\"true\",\"hadoop.registry.jaas.context\":\"Client\",\"yarn.timeline-service.leveldb-timeline-store.path\":\"${hadoop.tmp.dir}/yarn/timeline\",\"io.map.index.interval\":\"128\",\"yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms\":\"100\",\"fs.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem\",\"mapreduce.job.counters.max\":\"120\",\"mapreduce.jobhistory.webapp.rest-csrf.enabled\":\"false\",\"yarn.timeline-service.store-class\":\"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore\",\"mapreduce.jobhistory.move.interval-ms\":\"180000\",\"fs.s3a.change.detection.version.required\":\"true\",\"yarn.nodemanager.localizer.fetch.thread-count\":\"4\",\"yarn.resourcemanager.scheduler.client.thread-count\":\"50\",\"hadoop.ssl.hostname.verifier\":\"DEFAULT\",\"yarn.timeline-service.leveldb-state-store.path\":\"${hadoop.tmp.dir}/yarn/timeline\",\"mapreduce.job.classloader\":\"false\",\"mapreduce.task.profile.map.params\":\"${mapreduce.task.profile.params}\",\"ipc.client.connect.timeout\":\"20000\",\"hadoop.security.auth_to_local.mechanism\":\"hadoop\",\"yarn.timeline-service.app-collector.linger-period.ms\":\"60000\",\"yarn.nm.liveness-monitor.expiry-interval-ms\":\"600000\",\"yarn.resourcemanager.reservation-system.planfollower.time-step\":\"1000\",\"yarn.resourcemanager.proxy.timeout.enabled\":\"true\",\"yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms\":\"600000\",\"yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed\":\"true\",\"yarn.webapp.api-service.enable\":\"false\",\"yarn.nodemanager.recovery.enabled\":\"false\",\"mapreduce.job.end-notification.retry.interval\":\"1000\",\"fs.du.interval\":\"600000\",\"fs.ftp.impl\":\"org.apache.hadoop.fs.ftp.FTPFileSystem\",\"yarn.nodemanager.container.stderr.tail.bytes\":\"4096\",\"yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled\":\"true\",\"hadoop.security.group.mapping.ldap.read.timeout.ms\":\"60000\",\"hadoop.security.groups.cache.warn.after.ms\":\"5000\",\"file.bytes-per-checksum\":\"512\",\"mapreduce.outputcommitter.factory.scheme.s3a\":\"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory\",\"hadoop.security.groups.cache.background.reload\":\"false\",\"yarn.nodemanager.container-monitor.enabled\":\"true\",\"yarn.nodemanager.elastic-memory-control.enabled\":\"false\",\"net.topology.script.number.args\":\"100\",\"mapreduce.task.merge.progress.records\":\"10000\",\"yarn.nodemanager.localizer.address\":\"${yarn.nodemanager.hostname}:8040\",\"yarn.timeline-service.keytab\":\"/etc/krb5.keytab\",\"mapreduce.reduce.shuffle.fetch.retry.timeout-ms\":\"30000\",\"yarn.resourcemanager.rm.container-allocation.expiry-interval-ms\":\"600000\",\"yarn.nodemanager.container-executor.exit-code-file.timeout-ms\":\"2000\",\"mapreduce.fileoutputcommitter.algorithm.version\":\"1\",\"yarn.resourcemanager.work-preserving-recovery.enabled\":\"true\",\"mapreduce.map.skip.maxrecords\":\"0\",\"yarn.sharedcache.root-dir\":\"/sharedcache\",\"fs.s3a.retry.throttle.limit\":\"20\",\"hadoop.http.authentication.type\":\"simple\",\"fs.viewfs.overload.scheme.target.oss.impl\":\"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem\",\"mapreduce.job.cache.limit.max-resources\":\"0\",\"mapreduce.task.userlog.limit.kb\":\"0\",\"ipc.[port_number].weighted-cost.handler\":\"1\",\"yarn.resourcemanager.scheduler.monitor.enable\":\"false\",\"ipc.client.connect.max.retries\":\"10\",\"hadoop.registry.zk.retry.times\":\"5\",\"yarn.nodemanager.resource-monitor.interval-ms\":\"3000\",\"yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices\":\"auto\",\"mapreduce.job.sharedcache.mode\":\"disabled\",\"yarn.nodemanager.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.shuffle.listen.queue.size\":\"128\",\"yarn.scheduler.configuration.mutation.acl-policy.class\":\"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy\",\"mapreduce.map.cpu.vcores\":\"1\",\"yarn.log-aggregation.file-formats\":\"TFile\",\"yarn.timeline-service.client.fd-retain-secs\":\"300\",\"fs.s3a.select.output.csv.field.delimiter\":\",\",\"yarn.nodemanager.health-checker.timeout-ms\":\"1200000\",\"hadoop.user.group.static.mapping.overrides\":\"dr.who=;\",\"fs.azure.sas.expiry.period\":\"90d\",\"fs.s3a.select.output.csv.record.delimiter\":\"\\\\n\",\"mapreduce.jobhistory.recovery.store.class\":\"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService\",\"fs.viewfs.overload.scheme.target.https.impl\":\"org.apache.hadoop.fs.http.HttpsFileSystem\",\"fs.s3a.s3guard.ddb.table.sse.enabled\":\"false\",\"yarn.resourcemanager.fail-fast\":\"${yarn.fail-fast}\",\"yarn.resourcemanager.proxy-user-privileges.enabled\":\"false\",\"yarn.router.webapp.interceptor-class.pipeline\":\"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST\",\"yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage\":\"90.0\",\"mapreduce.job.reducer.preempt.delay.sec\":\"0\",\"hadoop.util.hash.type\":\"murmur\",\"yarn.nodemanager.disk-validator\":\"basic\",\"yarn.app.mapreduce.client.job.max-retries\":\"3\",\"fs.viewfs.overload.scheme.target.ftp.impl\":\"org.apache.hadoop.fs.ftp.FTPFileSystem\",\"mapreduce.reduce.shuffle.retry-delay.max.ms\":\"60000\",\"hadoop.security.group.mapping.ldap.connection.timeout.ms\":\"60000\",\"mapreduce.task.profile.params\":\"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s\",\"yarn.app.mapreduce.shuffle.log.backups\":\"0\",\"yarn.nodemanager.container-diagnostics-maximum-size\":\"10000\",\"hadoop.registry.zk.retry.interval.ms\":\"1000\",\"yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms\":\"1000\",\"fs.AbstractFileSystem.file.impl\":\"org.apache.hadoop.fs.local.LocalFs\",\"yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds\":\"-1\",\"mapreduce.jobhistory.cleaner.interval-ms\":\"86400000\",\"hadoop.registry.zk.quorum\":\"localhost:2181\",\"yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes\":\"runc\",\"mapreduce.output.fileoutputformat.compress\":\"false\",\"yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"fs.s3a.assumed.role.session.duration\":\"30m\",\"hadoop.security.group.mapping.ldap.conversion.rule\":\"none\",\"hadoop.ssl.server.conf\":\"ssl-server.xml\",\"fs.s3a.retry.throttle.interval\":\"100ms\",\"seq.io.sort.factor\":\"100\",\"fs.viewfs.overload.scheme.target.ofs.impl\":\"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem\",\"yarn.sharedcache.cleaner.initial-delay-mins\":\"10\",\"mapreduce.client.completion.pollinterval\":\"5000\",\"hadoop.ssl.keystores.factory.class\":\"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory\",\"yarn.app.mapreduce.am.resource.cpu-vcores\":\"1\",\"yarn.timeline-service.enabled\":\"false\",\"yarn.nodemanager.runtime.linux.docker.capabilities\":\"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE\",\"yarn.acl.enable\":\"false\",\"yarn.timeline-service.entity-group-fs-store.done-dir\":\"/tmp/entity-file-history/done/\",\"hadoop.security.group.mapping.ldap.num.attempts.before.failover\":\"3\",\"mapreduce.task.profile\":\"false\",\"hadoop.prometheus.endpoint.enabled\":\"false\",\"yarn.resourcemanager.fs.state-store.uri\":\"${hadoop.tmp.dir}/yarn/system/rmstore\",\"mapreduce.jobhistory.always-scan-user-dir\":\"false\",\"fs.s3a.metadatastore.metadata.ttl\":\"15m\",\"yarn.nodemanager.opportunistic-containers-use-pause-for-preemption\":\"false\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user\":\"nobody\",\"yarn.timeline-service.reader.class\":\"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl\",\"yarn.resourcemanager.configuration.provider-class\":\"org.apache.hadoop.yarn.LocalConfigurationProvider\",\"yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold\":\"1\",\"yarn.resourcemanager.configuration.file-system-based-store\":\"/yarn/conf\",\"mapreduce.job.cache.limit.max-single-resource-mb\":\"0\",\"yarn.nodemanager.runtime.linux.docker.stop.grace-period\":\"10\",\"yarn.resourcemanager.resource-profiles.source-file\":\"resource-profiles.json\",\"mapreduce.job.dfs.storage.capacity.kill-limit-exceed\":\"false\",\"yarn.nodemanager.resource.percentage-physical-cpu-limit\":\"100\",\"mapreduce.jobhistory.client.thread-count\":\"10\",\"tfile.fs.input.buffer.size\":\"262144\",\"mapreduce.client.progressmonitor.pollinterval\":\"1000\",\"yarn.nodemanager.log-dirs\":\"${yarn.log.dir}/userlogs\",\"yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat\":\"-1\",\"fs.automatic.close\":\"true\",\"yarn.resourcemanager.delegation-token-renewer.thread-retry-interval\":\"*********(redacted)\",\"fs.s3a.select.input.csv.quote.character\":\"\\\"\",\"yarn.nodemanager.hostname\":\"0.0.0.0\",\"ipc.[port_number].cost-provider.impl\":\"org.apache.hadoop.ipc.DefaultCostProvider\",\"yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin\",\"yarn.nodemanager.remote-app-log-dir-include-older\":\"true\",\"yarn.nodemanager.resource.memory.cgroups.swappiness\":\"0\",\"ftp.stream-buffer-size\":\"4096\",\"yarn.fail-fast\":\"false\",\"yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep\":\"100\",\"yarn.timeline-service.app-aggregation-interval-secs\":\"15\",\"hadoop.security.group.mapping.ldap.search.filter.user\":\"(&(objectClass=user)(sAMAccountName={0}))\",\"ipc.[port_number].weighted-cost.lockshared\":\"10\",\"yarn.nodemanager.container-localizer.log.level\":\"INFO\",\"yarn.timeline-service.address\":\"${yarn.timeline-service.hostname}:10200\",\"mapreduce.job.ubertask.maxmaps\":\"9\",\"fs.s3a.threads.keepalivetime\":\"60\",\"mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"mapreduce.task.files.preserve.failedtasks\":\"false\",\"yarn.app.mapreduce.client.job.retry-interval\":\"2000\",\"ha.failover-controller.graceful-fence.connection.retries\":\"1\",\"fs.s3a.select.output.csv.quote.escape.character\":\"\\\\\\\\\",\"yarn.resourcemanager.delegation.token.max-lifetime\":\"*********(redacted)\",\"hadoop.kerberos.keytab.login.autorenewal.enabled\":\"false\",\"yarn.timeline-service.client.drain-entities.timeout.ms\":\"2000\",\"yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms\":\"1000\",\"yarn.timeline-service.entity-group-fs-store.summary-store\":\"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore\",\"mapreduce.reduce.cpu.vcores\":\"1\",\"mapreduce.job.encrypted-intermediate-data.buffer.kb\":\"128\",\"fs.client.resolve.remote.symlinks\":\"true\",\"yarn.nodemanager.webapp.https.address\":\"0.0.0.0:8044\",\"hadoop.http.cross-origin.allowed-origins\":\"*\",\"mapreduce.job.encrypted-intermediate-data\":\"false\",\"yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled\":\"true\",\"fs.s3a.executor.capacity\":\"16\",\"yarn.timeline-service.entity-group-fs-store.retain-seconds\":\"604800\",\"yarn.resourcemanager.metrics.runtime.buckets\":\"60,300,1440\",\"yarn.timeline-service.generic-application-history.max-applications\":\"10000\",\"yarn.nodemanager.local-dirs\":\"${hadoop.tmp.dir}/nm-local-dir\",\"mapreduce.shuffle.connection-keep-alive.enable\":\"false\",\"yarn.node-labels.configuration-type\":\"centralized\",\"fs.s3a.path.style.access\":\"false\",\"yarn.nodemanager.aux-services.mapreduce_shuffle.class\":\"org.apache.hadoop.mapred.ShuffleHandler\",\"yarn.sharedcache.store.in-memory.staleness-period-mins\":\"10080\",\"fs.adl.impl\":\"org.apache.hadoop.fs.adl.AdlFileSystem\",\"yarn.resourcemanager.application.max-tags\":\"10\",\"hadoop.domainname.resolver.impl\":\"org.apache.hadoop.net.DNSDomainNameResolver\",\"yarn.resourcemanager.nodemanager.minimum.version\":\"NONE\",\"mapreduce.jobhistory.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled\":\"false\",\"net.topology.impl\":\"org.apache.hadoop.net.NetworkTopology\",\"io.map.index.skip\":\"0\",\"yarn.timeline-service.reader.webapp.https.address\":\"${yarn.timeline-service.webapp.https.address}\",\"fs.ftp.data.connection.mode\":\"ACTIVE_LOCAL_DATA_CONNECTION_MODE\",\"mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed\":\"true\",\"fs.azure.buffer.dir\":\"${hadoop.tmp.dir}/abfs\",\"yarn.scheduler.maximum-allocation-vcores\":\"4\",\"hadoop.http.cross-origin.allowed-headers\":\"X-Requested-With,Content-Type,Accept,Origin\",\"yarn.nodemanager.log-aggregation.compression-type\":\"none\",\"yarn.timeline-service.version\":\"1.0f\",\"yarn.ipc.rpc.class\":\"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC\",\"mapreduce.reduce.maxattempts\":\"4\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size\":\"1000\",\"hadoop.security.dns.log-slow-lookups.enabled\":\"false\",\"mapreduce.job.committer.setup.cleanup.needed\":\"true\",\"hadoop.security.secure.random.impl\":\"org.apache.hadoop.crypto.random.OpensslSecureRandom\",\"mapreduce.job.running.reduce.limit\":\"0\",\"fs.s3a.select.errors.include.sql\":\"false\",\"fs.s3a.connection.request.timeout\":\"0\",\"ipc.maximum.response.length\":\"134217728\",\"yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"mapreduce.job.token.tracking.ids.enabled\":\"*********(redacted)\",\"hadoop.caller.context.max.size\":\"128\",\"yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed\":\"false\",\"yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed\":\"false\",\"hadoop.registry.system.acls\":\"sasl:yarn@, sasl:mapred@, sasl:hdfs@\",\"yarn.nodemanager.recovery.dir\":\"${hadoop.tmp.dir}/yarn-nm-recovery\",\"fs.s3a.fast.upload.buffer\":\"disk\",\"mapreduce.jobhistory.intermediate-done-dir\":\"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate\",\"yarn.app.mapreduce.shuffle.log.separate\":\"true\",\"yarn.log-aggregation.debug.filesize\":\"104857600\",\"fs.s3a.max.total.tasks\":\"32\",\"fs.s3a.readahead.range\":\"64K\",\"hadoop.http.authentication.simple.anonymous.allowed\":\"true\",\"fs.s3a.attempts.maximum\":\"20\",\"hadoop.registry.zk.connection.timeout.ms\":\"15000\",\"yarn.resourcemanager.delegation-token-renewer.thread-count\":\"*********(redacted)\",\"yarn.resourcemanager.delegation-token-renewer.thread-timeout\":\"*********(redacted)\",\"yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size\":\"10000\",\"yarn.nodemanager.aux-services.manifest.reload-ms\":\"0\",\"yarn.nodemanager.emit-container-events\":\"true\",\"yarn.resourcemanager.resource-profiles.enabled\":\"false\",\"yarn.timeline-service.hbase-schema.prefix\":\"prod.\",\"fs.azure.authorization\":\"false\",\"mapreduce.map.log.level\":\"INFO\",\"ha.failover-controller.active-standby-elector.zk.op.retries\":\"3\",\"yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs\":\"20\",\"mapreduce.output.fileoutputformat.compress.type\":\"RECORD\",\"yarn.resourcemanager.leveldb-state-store.path\":\"${hadoop.tmp.dir}/yarn/system/rmstore\",\"yarn.timeline-service.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"mapreduce.ifile.readahead.bytes\":\"4194304\",\"yarn.sharedcache.app-checker.class\":\"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker\",\"yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users\":\"true\",\"yarn.nodemanager.resource.detect-hardware-capabilities\":\"false\",\"mapreduce.cluster.acls.enabled\":\"false\",\"mapreduce.job.speculative.retry-after-no-speculate\":\"1000\",\"fs.viewfs.overload.scheme.target.abfs.impl\":\"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem\",\"hadoop.security.group.mapping.ldap.search.group.hierarchy.levels\":\"0\",\"yarn.resourcemanager.fs.state-store.retry-interval-ms\":\"1000\",\"file.stream-buffer-size\":\"4096\",\"yarn.resourcemanager.application-timeouts.monitor.interval-ms\":\"3000\",\"mapreduce.map.output.compress.codec\":\"org.apache.hadoop.io.compress.DefaultCodec\",\"mapreduce.map.speculative\":\"true\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file\":\"/runc-root/image-tag-to-hash\",\"mapreduce.job.speculative.retry-after-speculate\":\"15000\",\"yarn.nodemanager.linux-container-executor.cgroups.mount\":\"false\",\"yarn.app.mapreduce.am.container.log.backups\":\"0\",\"yarn.app.mapreduce.am.log.level\":\"INFO\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin\":\"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin\",\"io.bytes.per.checksum\":\"512\",\"mapreduce.job.reduce.slowstart.completedmaps\":\"0.05\",\"yarn.timeline-service.http-authentication.type\":\"simple\",\"hadoop.security.group.mapping.ldap.search.attr.group.name\":\"cn\",\"yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices\":\"auto\",\"yarn.timeline-service.client.internal-timers-ttl-secs\":\"420\",\"fs.s3a.select.output.csv.quote.character\":\"\\\"\",\"hadoop.http.logs.enabled\":\"true\",\"fs.s3a.block.size\":\"32M\",\"yarn.sharedcache.client-server.address\":\"0.0.0.0:8045\",\"yarn.nodemanager.logaggregation.threadpool-size-max\":\"100\",\"yarn.resourcemanager.hostname\":\"0.0.0.0\",\"yarn.resourcemanager.delegation.key.update-interval\":\"86400000\",\"mapreduce.reduce.shuffle.fetch.retry.enabled\":\"${yarn.nodemanager.recovery.enabled}\",\"mapreduce.map.memory.mb\":\"-1\",\"mapreduce.task.skip.start.attempts\":\"2\",\"fs.AbstractFileSystem.hdfs.impl\":\"org.apache.hadoop.fs.Hdfs\",\"yarn.nodemanager.disk-health-checker.enable\":\"true\",\"fs.s3a.select.output.csv.quote.fields\":\"always\",\"ipc.client.tcpnodelay\":\"true\",\"ipc.client.rpc-timeout.ms\":\"0\",\"yarn.nodemanager.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts\":\"*********(redacted)\",\"ipc.client.low-latency\":\"false\",\"mapreduce.input.lineinputformat.linespermap\":\"1\",\"yarn.router.interceptor.user.threadpool-size\":\"5\",\"ipc.client.connect.max.retries.on.timeouts\":\"45\",\"yarn.timeline-service.leveldb-timeline-store.read-cache-size\":\"104857600\",\"fs.AbstractFileSystem.har.impl\":\"org.apache.hadoop.fs.HarFs\",\"mapreduce.job.split.metainfo.maxsize\":\"10000000\",\"yarn.am.liveness-monitor.expiry-interval-ms\":\"600000\",\"yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs\":\"*********(redacted)\",\"yarn.timeline-service.entity-group-fs-store.app-cache-size\":\"10\",\"yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs\":\"360\",\"fs.s3a.socket.recv.buffer\":\"8192\",\"rpc.metrics.timeunit\":\"MILLISECONDS\",\"yarn.resourcemanager.resource-tracker.address\":\"${yarn.resourcemanager.hostname}:8031\",\"yarn.nodemanager.node-labels.provider.fetch-timeout-ms\":\"1200000\",\"mapreduce.job.heap.memory-mb.ratio\":\"0.8\",\"yarn.resourcemanager.leveldb-state-store.compaction-interval-secs\":\"3600\",\"yarn.resourcemanager.webapp.rest-csrf.custom-header\":\"X-XSRF-Header\",\"yarn.nodemanager.pluggable-device-framework.enabled\":\"false\",\"yarn.scheduler.configuration.fs.path\":\"file://${hadoop.tmp.dir}/yarn/system/schedconf\",\"mapreduce.client.output.filter\":\"FAILED\",\"hadoop.http.filter.initializers\":\"org.apache.hadoop.http.lib.StaticUserWebFilter\",\"mapreduce.reduce.memory.mb\":\"-1\",\"yarn.timeline-service.hostname\":\"0.0.0.0\",\"file.replication\":\"1\",\"yarn.nodemanager.container-metrics.unregister-delay-ms\":\"10000\",\"yarn.nodemanager.container-metrics.period-ms\":\"-1\",\"mapreduce.fileoutputcommitter.task.cleanup.enabled\":\"false\",\"yarn.nodemanager.log.retain-seconds\":\"10800\",\"yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds\":\"3600\",\"ipc.[port_number].callqueue.impl\":\"java.util.concurrent.LinkedBlockingQueue\",\"yarn.resourcemanager.keytab\":\"/etc/krb5.keytab\",\"hadoop.security.group.mapping.providers.combined\":\"true\",\"mapreduce.reduce.merge.inmem.threshold\":\"1000\",\"yarn.timeline-service.recovery.enabled\":\"false\",\"fs.azure.saskey.usecontainersaskeyforallaccess\":\"true\",\"yarn.sharedcache.nm.uploader.thread-count\":\"20\",\"yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs\":\"3600\",\"ipc.[port_number].weighted-cost.lockfree\":\"1\",\"mapreduce.shuffle.ssl.enabled\":\"false\",\"yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds\":\"259200000\",\"yarn.nodemanager.opportunistic-containers-max-queue-length\":\"0\",\"yarn.resourcemanager.state-store.max-completed-applications\":\"${yarn.resourcemanager.max-completed-applications}\",\"mapreduce.job.speculative.minimum-allowed-tasks\":\"10\",\"fs.s3a.aws.credentials.provider\":\"\\n org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\\n org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\\n com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\\n org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\\n \",\"yarn.log-aggregation.retain-seconds\":\"-1\",\"yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb\":\"0\",\"mapreduce.jobhistory.max-age-ms\":\"604800000\",\"hadoop.http.cross-origin.allowed-methods\":\"GET,POST,HEAD\",\"yarn.resourcemanager.opportunistic-container-allocation.enabled\":\"false\",\"mapreduce.jobhistory.webapp.address\":\"0.0.0.0:19888\",\"hadoop.system.tags\":\"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\\n ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL\",\"yarn.log-aggregation.file-controller.TFile.class\":\"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController\",\"yarn.client.nodemanager-connect.max-wait-ms\":\"180000\",\"yarn.resourcemanager.webapp.address\":\"${yarn.resourcemanager.hostname}:8088\",\"mapreduce.jobhistory.recovery.enable\":\"false\",\"mapreduce.reduce.shuffle.parallelcopies\":\"5\",\"fs.AbstractFileSystem.webhdfs.impl\":\"org.apache.hadoop.fs.WebHdfs\",\"fs.trash.interval\":\"0\",\"yarn.app.mapreduce.client.max-retries\":\"3\",\"hadoop.security.authentication\":\"simple\",\"mapreduce.task.profile.reduce.params\":\"${mapreduce.task.profile.params}\",\"yarn.app.mapreduce.am.resource.mb\":\"1536\",\"mapreduce.input.fileinputformat.list-status.num-threads\":\"1\",\"yarn.nodemanager.container-executor.class\":\"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor\",\"io.mapfile.bloom.size\":\"1048576\",\"yarn.timeline-service.ttl-ms\":\"604800000\",\"yarn.resourcemanager.nm-container-queuing.min-queue-length\":\"5\",\"yarn.nodemanager.resource.cpu-vcores\":\"-1\",\"mapreduce.job.reduces\":\"1\",\"fs.s3a.multipart.size\":\"64M\",\"fs.s3a.select.input.csv.comment.marker\":\"#\",\"yarn.scheduler.minimum-allocation-vcores\":\"1\",\"mapreduce.job.speculative.speculative-cap-total-tasks\":\"0.01\",\"hadoop.ssl.client.conf\":\"ssl-client.xml\",\"mapreduce.job.queuename\":\"default\",\"mapreduce.job.encrypted-intermediate-data-key-size-bits\":\"128\",\"fs.s3a.metadatastore.authoritative\":\"false\",\"ipc.[port_number].weighted-cost.response\":\"1\",\"yarn.nodemanager.webapp.xfs-filter.xframe-options\":\"SAMEORIGIN\",\"ha.health-monitor.sleep-after-disconnect.ms\":\"1000\",\"yarn.app.mapreduce.shuffle.log.limit.kb\":\"0\",\"hadoop.security.group.mapping\":\"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback\",\"yarn.client.application-client-protocol.poll-timeout-ms\":\"-1\",\"mapreduce.jobhistory.jhist.format\":\"binary\",\"mapreduce.task.stuck.timeout-ms\":\"600000\",\"yarn.resourcemanager.application.max-tag.length\":\"100\",\"yarn.resourcemanager.ha.enabled\":\"false\",\"dfs.client.ignore.namenode.default.kms.uri\":\"false\",\"hadoop.http.staticuser.user\":\"dr.who\",\"mapreduce.task.exit.timeout.check-interval-ms\":\"20000\",\"mapreduce.jobhistory.intermediate-user-done-dir.permissions\":\"770\",\"mapreduce.task.exit.timeout\":\"60000\",\"yarn.nodemanager.linux-container-executor.resources-handler.class\":\"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler\",\"mapreduce.reduce.shuffle.memory.limit.percent\":\"0.25\",\"yarn.resourcemanager.reservation-system.enable\":\"false\",\"mapreduce.map.output.compress\":\"false\",\"ha.zookeeper.acl\":\"world:anyone:rwcda\",\"ipc.server.max.connections\":\"0\",\"yarn.nodemanager.runtime.linux.docker.default-container-network\":\"host\",\"yarn.router.webapp.address\":\"0.0.0.0:8089\",\"yarn.scheduler.maximum-allocation-mb\":\"8192\",\"yarn.resourcemanager.scheduler.monitor.policies\":\"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy\",\"yarn.sharedcache.cleaner.period-mins\":\"1440\",\"yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint\":\"http://localhost:3476/v1.0/docker/cli\",\"yarn.app.mapreduce.am.container.log.limit.kb\":\"0\",\"ipc.client.connect.retry.interval\":\"1000\",\"yarn.timeline-service.http-cross-origin.enabled\":\"false\",\"fs.wasbs.impl\":\"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure\",\"yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms\":\"1000\",\"yarn.federation.subcluster-resolver.class\":\"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl\",\"yarn.resourcemanager.zk-state-store.parent-path\":\"/rmstore\",\"fs.s3a.select.input.csv.field.delimiter\":\",\",\"mapreduce.jobhistory.cleaner.enable\":\"true\",\"yarn.timeline-service.client.fd-flush-interval-secs\":\"10\",\"hadoop.security.kms.client.encrypted.key.cache.expiry\":\"43200000\",\"yarn.client.nodemanager-client-async.thread-pool-max-size\":\"500\",\"mapreduce.map.maxattempts\":\"4\",\"yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms\":\"1000\",\"fs.s3a.committer.staging.tmp.path\":\"tmp/staging\",\"yarn.nodemanager.sleep-delay-before-sigkill.ms\":\"250\",\"yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms\":\"10\",\"mapreduce.job.end-notification.retry.attempts\":\"0\",\"yarn.nodemanager.resource.count-logical-processors-as-cores\":\"false\",\"hadoop.registry.zk.root\":\"/registry\",\"adl.feature.ownerandgroup.enableupn\":\"false\",\"yarn.resourcemanager.zk-max-znode-size.bytes\":\"1048576\",\"mapreduce.job.reduce.shuffle.consumer.plugin.class\":\"org.apache.hadoop.mapreduce.task.reduce.Shuffle\",\"yarn.resourcemanager.delayed.delegation-token.removal-interval-ms\":\"*********(redacted)\",\"yarn.nodemanager.localizer.cache.target-size-mb\":\"10240\",\"fs.s3a.committer.staging.conflict-mode\":\"append\",\"mapreduce.client.libjars.wildcard\":\"true\",\"fs.s3a.committer.staging.unique-filenames\":\"true\",\"yarn.nodemanager.node-attributes.provider.fetch-timeout-ms\":\"1200000\",\"fs.s3a.list.version\":\"2\",\"ftp.client-write-packet-size\":\"65536\",\"ipc.[port_number].weighted-cost.lockexclusive\":\"100\",\"fs.AbstractFileSystem.adl.impl\":\"org.apache.hadoop.fs.adl.Adl\",\"yarn.nodemanager.container-log-monitor.enable\":\"false\",\"hadoop.security.key.default.cipher\":\"AES/CTR/NoPadding\",\"yarn.client.failover-retries\":\"0\",\"fs.s3a.multipart.purge.age\":\"86400\",\"mapreduce.job.local-fs.single-disk-limit.check.interval-ms\":\"5000\",\"net.topology.node.switch.mapping.impl\":\"org.apache.hadoop.net.ScriptBasedMapping\",\"yarn.nodemanager.amrmproxy.address\":\"0.0.0.0:8049\",\"ipc.server.listen.queue.size\":\"256\",\"ipc.[port_number].decay-scheduler.period-ms\":\"5000\",\"yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs\":\"60\",\"map.sort.class\":\"org.apache.hadoop.util.QuickSort\",\"fs.viewfs.rename.strategy\":\"SAME_MOUNTPOINT\",\"hadoop.security.kms.client.authentication.retry-count\":\"1\",\"fs.permissions.umask-mode\":\"022\",\"fs.s3a.assumed.role.credentials.provider\":\"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\",\"yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed\":\"false\",\"yarn.nodemanager.vmem-check-enabled\":\"true\",\"yarn.nodemanager.numa-awareness.enabled\":\"false\",\"yarn.nodemanager.recovery.compaction-interval-secs\":\"3600\",\"yarn.app.mapreduce.client-am.ipc.max-retries\":\"3\",\"yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds\":\"60\",\"yarn.federation.registry.base-dir\":\"yarnfederation/\",\"yarn.nodemanager.health-checker.run-before-startup\":\"false\",\"mapreduce.job.max.map\":\"-1\",\"mapreduce.job.local-fs.single-disk-limit.bytes\":\"-1\",\"mapreduce.shuffle.pathcache.concurrency-level\":\"16\",\"mapreduce.job.ubertask.maxreduces\":\"1\",\"mapreduce.shuffle.pathcache.max-weight\":\"10485760\",\"hadoop.security.kms.client.encrypted.key.cache.size\":\"500\",\"hadoop.security.java.secure.random.algorithm\":\"SHA1PRNG\",\"ha.failover-controller.cli-check.rpc-timeout.ms\":\"20000\",\"mapreduce.jobhistory.jobname.limit\":\"50\",\"fs.s3a.select.input.compression\":\"none\",\"yarn.client.nodemanager-connect.retry-interval-ms\":\"10000\",\"ipc.[port_number].scheduler.priority.levels\":\"4\",\"yarn.timeline-service.state-store-class\":\"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore\",\"yarn.nodemanager.env-whitelist\":\"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ\",\"yarn.sharedcache.nested-level\":\"3\",\"yarn.timeline-service.webapp.rest-csrf.methods-to-ignore\":\"GET,OPTIONS,HEAD\",\"fs.azure.user.agent.prefix\":\"unknown\",\"yarn.resourcemanager.zk-delegation-token-node.split-index\":\"*********(redacted)\",\"yarn.nodemanager.numa-awareness.read-topology\":\"false\",\"yarn.nodemanager.webapp.address\":\"${yarn.nodemanager.hostname}:8042\",\"rpc.metrics.quantile.enable\":\"false\",\"yarn.registry.class\":\"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService\",\"mapreduce.jobhistory.admin.acl\":\"*\",\"yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size\":\"10\",\"yarn.scheduler.queue-placement-rules\":\"user-group\",\"hadoop.http.authentication.kerberos.keytab\":\"${user.home}/hadoop.keytab\",\"yarn.resourcemanager.recovery.enabled\":\"false\",\"fs.s3a.select.input.csv.header\":\"none\",\"yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size\":\"500\",\"yarn.timeline-service.webapp.rest-csrf.enabled\":\"false\",\"yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb\":\"0\"},\"System Properties\":{\"java.io.tmpdir\":\"/tmp\",\"line.separator\":\"\\n\",\"path.separator\":\":\",\"sun.management.compiler\":\"HotSpot 64-Bit Tiered Compilers\",\"SPARK_SUBMIT\":\"true\",\"sun.cpu.endian\":\"little\",\"java.specification.maintenance.version\":\"2\",\"java.specification.version\":\"11\",\"java.vm.specification.name\":\"Java Virtual Machine Specification\",\"java.vendor\":\"Ubuntu\",\"java.vm.specification.version\":\"11\",\"user.home\":\"/root\",\"sun.arch.data.model\":\"64\",\"sun.boot.library.path\":\"/usr/lib/jvm/java-11-openjdk-amd64/lib\",\"user.dir\":\"/content\",\"java.library.path\":\"/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib\",\"sun.cpu.isalist\":\"\",\"os.arch\":\"amd64\",\"java.vm.version\":\"11.0.24+8-post-Ubuntu-1ubuntu322.04\",\"jetty.git.hash\":\"cef3fbd6d736a21e7d541a5db490381d95a2047d\",\"java.runtime.version\":\"11.0.24+8-post-Ubuntu-1ubuntu322.04\",\"java.vm.info\":\"mixed mode, sharing\",\"java.runtime.name\":\"OpenJDK Runtime Environment\",\"java.version.date\":\"2024-07-16\",\"file.separator\":\"/\",\"java.class.version\":\"55.0\",\"java.specification.name\":\"Java Platform API Specification\",\"file.encoding\":\"UTF-8\",\"jdk.reflect.useDirectMethodHandle\":\"false\",\"user.timezone\":\"Etc/UTC\",\"java.specification.vendor\":\"Oracle Corporation\",\"sun.java.launcher\":\"SUN_STANDARD\",\"java.vm.compressedOopsMode\":\"32-bit\",\"os.version\":\"6.1.85+\",\"sun.os.patch.level\":\"unknown\",\"java.vm.specification.vendor\":\"Oracle Corporation\",\"user.country\":\"US\",\"sun.jnu.encoding\":\"UTF-8\",\"user.language\":\"en\",\"java.vendor.url\":\"https://ubuntu.com/\",\"java.awt.printerjob\":\"sun.print.PSPrinterJob\",\"java.awt.graphicsenv\":\"sun.awt.X11GraphicsEnvironment\",\"awt.toolkit\":\"sun.awt.X11.XToolkit\",\"os.name\":\"Linux\",\"java.vm.vendor\":\"Ubuntu\",\"jdk.debug\":\"release\",\"java.vendor.url.bug\":\"https://bugs.launchpad.net/ubuntu/+source/openjdk-lts\",\"user.name\":\"root\",\"java.vm.name\":\"OpenJDK 64-Bit Server VM\",\"sun.java.command\":\"org.apache.spark.deploy.SparkSubmit --master spark://651dec53631c:7077 --conf spark.eventLog.enabled=true --class org.apache.spark.examples.JavaWordCount /content/spark-3.5.3-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.3.jar datafile.txt\",\"java.home\":\"/usr/lib/jvm/java-11-openjdk-amd64\",\"java.version\":\"11.0.24\",\"sun.io.unicode.encoding\":\"UnicodeLittle\"},\"Metrics Properties\":{\"*.sink.servlet.class\":\"org.apache.spark.metrics.sink.MetricsServlet\",\"*.sink.servlet.path\":\"/metrics/json\",\"applications.sink.servlet.path\":\"/metrics/applications/json\",\"master.sink.servlet.path\":\"/metrics/master/json\"},\"Classpath Entries\":{\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-resource-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/chill_2.12-0.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-serde-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/paranamer-2.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-collections4-4.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-dataformat-yaml-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-cli-1.5.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/joda-time-2.12.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-jvm-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-repl_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-math3-3.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-all-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-graphx_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-exec-2.3.9-core.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arpack_combined_all-0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jcl-over-slf4j-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kryo-shaded-4.0.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arpack-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/rocksdbjni-8.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-extensions-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zstd-jni-1.5.5-4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/lapack-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-certificates-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/okio-1.17.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-client-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/ST4-4.0.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sql_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-metrics-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/pickle-1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jline-2.14.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/leveldbjni-all-1.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-core-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-unsafe_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zjsonpatch-0.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-netty-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/guava-14.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-launcher_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-container-servlet-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-locator-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/xbean-asm9-shaded-4.23.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-lang-2.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-logging-1.1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/shims-0.9.45.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datasketches-java-3.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-core-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/libfb303-0.9.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/annotations-17.0.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-kvstore_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-batch-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-slf4j2-impl-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/libthrift-0.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-metastore-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-framework-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-networking-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-core_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-codec-1.16.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/tink-1.9.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-1.2-api-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-compress-1.23.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-json-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-compiler-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-coordination-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-common-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/derby-10.14.2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/super-csv-2.2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-io-2.16.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/conf/\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/transaction-api-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-text-1.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/xz-1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/ivy-2.5.1.jar\":\"System Classpath\",\"spark://651dec53631c:33373/jars/spark-examples_2.12-3.5.3.jar\":\"Added By User\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.inject-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jdo-api-3.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-core-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/blas-3.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/httpclient-4.5.14.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datasketches-memory-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-engine-2.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-common-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/aircompressor-0.27.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-events-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-tags_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snappy-java-1.1.10.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/opencsv-2.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-core-1.9.4-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-beeline-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/audience-annotations-0.5.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-jmx-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sql-api_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-compiler-3.1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jpam-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/stream-2.9.6.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mesos_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-common-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-common-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/JLargeArrays-1.5.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-scheduling-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-resolver-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javassist-3.29.2-GA.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-mapred-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/minlog-1.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jta-1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/chill-java-0.10.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-node-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-format-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/compress-lzf-1.1.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/objenesis-3.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-server-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-shims-1.9.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/snakeyaml-2.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-cli-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-encoding-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-api-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/activation-1.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-column-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-api-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json-1.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jsr305-3.0.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/JTransforms-3.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/stax-api-1.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-handler-proxy-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-databind-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-graphite-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/gson-2.2.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jul-to-slf4j-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-policy-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/antlr-runtime-3.5.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/breeze_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-lang3-3.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-network-common_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-common-utils_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-parser-combinators_2.12-2.3.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-llap-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-jackson-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-reflect-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-dbcp-1.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/logging-interceptor-3.12.12.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-core-4.1.17.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-buffer-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/univocity-parsers-2.9.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/log4j-api-2.20.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jersey-hk2-2.40.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-sketch_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-hive_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-storage-api-2.8.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-annotations-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-client-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-storageclass-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-kubernetes_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-http2-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/httpcore-4.4.16.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/oro-2.0.8.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/orc-mapreduce-1.9.4-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-client-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hk2-utils-2.6.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-format-structures-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/okhttp-3.12.12.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/metrics-core-4.2.19.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/threeten-extra-1.7.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-yarn_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-crypto-1.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-shims-common-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/curator-recipes-2.13.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-jdbc-2.3.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-core-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-autoscaling-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/RoaringBitmap-0.9.45.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/HikariCP-2.5.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-vector-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-pool-1.5.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-catalyst_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jodd-core-3.5.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-discovery-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/commons-collections-3.2.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/zookeeper-3.6.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javolution-5.5.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/slf4j-api-2.0.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/py4j-0.10.9.7.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-mllib-local_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/lz4-java-1.8.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-module-scala_2.12-2.15.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-network-shuffle_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-codec-socks-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/avro-ipc-1.11.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/spark-streaming_2.12-3.5.3.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/arrow-memory-core-12.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/algebra_2.12-2.0.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apiextensions-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-apps-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/janino-3.1.9.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/scala-library-2.12.18.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.96.Final.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/kubernetes-model-rbac-6.7.2.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/parquet-hadoop-1.13.1.jar\":\"System Classpath\",\"/content/spark-3.5.3-bin-hadoop3/jars/jackson-datatype-jsr310-2.15.2.jar\":\"System Classpath\"}}\n",
"{\"Event\":\"SparkListenerApplicationStart\",\"App Name\":\"JavaWordCount\",\"App ID\":\"app-20241007115053-0001\",\"Timestamp\":1728301851395,\"User\":\"root\"}\n",
"{\"Event\":\"SparkListenerExecutorAdded\",\"Timestamp\":1728301863574,\"Executor ID\":\"0\",\"Executor Info\":{\"Host\":\"172.28.0.12\",\"Total Cores\":2,\"Log Urls\":{\"stdout\":\"http://172.28.0.12:8082/logPage/?appId=app-20241007115053-0001&executorId=0&logType=stdout\",\"stderr\":\"http://172.28.0.12:8082/logPage/?appId=app-20241007115053-0001&executorId=0&logType=stderr\"},\"Attributes\":{},\"Resources\":{},\"Resource Profile Id\":0,\"Registration Time\":1728301863574}}\n",
"{\"Event\":\"SparkListenerBlockManagerAdded\",\"Block Manager ID\":{\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Port\":35809},\"Maximum Memory\":455501414,\"Timestamp\":1728301863844,\"Maximum Onheap Memory\":455501414,\"Maximum Offheap Memory\":0}\n",
"{\"Event\":\"SparkListenerJobStart\",\"Job ID\":0,\"Submission Time\":1728301867496,\"Stage Infos\":[{\"Stage ID\":0,\"Stage Attempt ID\":0,\"Stage Name\":\"mapToPair at JavaWordCount.java:49\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":6,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"4\\\",\\\"name\\\":\\\"map\\\"}\",\"Callsite\":\"mapToPair at JavaWordCount.java:49\",\"Parent IDs\":[5],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":0,\"Name\":\"FileScanRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":4,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"2\\\",\\\"name\\\":\\\"mapPartitions\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[3],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":2,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"0\\\",\\\"name\\\":\\\"DeserializeToObject\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[1],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":3,\"Name\":\"SQLExecutionRDD\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[2],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":5,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"3\\\",\\\"name\\\":\\\"flatMap\\\"}\",\"Callsite\":\"flatMap at JavaWordCount.java:47\",\"Parent IDs\":[4],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":1,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[0],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0},{\"Stage ID\":1,\"Stage Attempt ID\":0,\"Stage Name\":\"collect at JavaWordCount.java:53\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":7,\"Name\":\"ShuffledRDD\",\"Scope\":\"{\\\"id\\\":\\\"5\\\",\\\"name\\\":\\\"reduceByKey\\\"}\",\"Callsite\":\"reduceByKey at JavaWordCount.java:51\",\"Parent IDs\":[6],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"UNORDERED\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[0],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0}],\"Stage IDs\":[0,1],\"Properties\":{\"spark.rdd.scope\":\"{\\\"id\\\":\\\"6\\\",\\\"name\\\":\\\"collect\\\"}\",\"spark.rdd.scope.noOverride\":\"true\"}}\n",
"{\"Event\":\"SparkListenerStageSubmitted\",\"Stage Info\":{\"Stage ID\":0,\"Stage Attempt ID\":0,\"Stage Name\":\"mapToPair at JavaWordCount.java:49\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":6,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"4\\\",\\\"name\\\":\\\"map\\\"}\",\"Callsite\":\"mapToPair at JavaWordCount.java:49\",\"Parent IDs\":[5],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":0,\"Name\":\"FileScanRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":4,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"2\\\",\\\"name\\\":\\\"mapPartitions\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[3],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":2,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"0\\\",\\\"name\\\":\\\"DeserializeToObject\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[1],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":3,\"Name\":\"SQLExecutionRDD\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[2],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":5,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"3\\\",\\\"name\\\":\\\"flatMap\\\"}\",\"Callsite\":\"flatMap at JavaWordCount.java:47\",\"Parent IDs\":[4],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":1,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[0],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Submission Time\":1728301867588,\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0},\"Properties\":{\"spark.rdd.scope\":\"{\\\"id\\\":\\\"6\\\",\\\"name\\\":\\\"collect\\\"}\",\"spark.rdd.scope.noOverride\":\"true\"}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":0,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301867803,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":0,\"Stage Attempt ID\":0,\"Task Type\":\"ShuffleMapTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":0,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301867803,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"PROCESS_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301873732,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":0,\"Name\":\"number of output rows\",\"Update\":\"921\",\"Value\":\"921\",\"Internal\":true,\"Count Failed Values\":true,\"Metadata\":\"sql\"},{\"ID\":4,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":1159,\"Value\":1159,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":5,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":864336864,\"Value\":864336864,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":4604,\"Value\":4604,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":7,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":2593529632,\"Value\":2593529632,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":8,\"Name\":\"internal.metrics.resultSize\",\"Update\":1842,\"Value\":1842,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":9,\"Name\":\"internal.metrics.jvmGCTime\",\"Update\":99,\"Value\":99,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":10,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":4,\"Value\":4,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":11,\"Name\":\"internal.metrics.memoryBytesSpilled\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":12,\"Name\":\"internal.metrics.diskBytesSpilled\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":13,\"Name\":\"internal.metrics.peakExecutionMemory\",\"Update\":219951,\"Value\":219951,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":32,\"Name\":\"internal.metrics.shuffle.write.bytesWritten\",\"Update\":21712,\"Value\":21712,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":33,\"Name\":\"internal.metrics.shuffle.write.recordsWritten\",\"Update\":2357,\"Value\":2357,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":34,\"Name\":\"internal.metrics.shuffle.write.writeTime\",\"Update\":1716215,\"Value\":1716215,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":35,\"Name\":\"internal.metrics.input.bytesRead\",\"Update\":47566,\"Value\":47566,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":36,\"Name\":\"internal.metrics.input.recordsRead\",\"Update\":921,\"Value\":921,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":1159,\"Executor Deserialize CPU Time\":864336864,\"Executor Run Time\":4604,\"Executor CPU Time\":2593529632,\"Peak Execution Memory\":219951,\"Result Size\":1842,\"JVM GC Time\":99,\"Result Serialization Time\":4,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":0,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":0,\"Total Records Read\":0,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":21712,\"Shuffle Write Time\":1716215,\"Shuffle Records Written\":2357},\"Input Metrics\":{\"Bytes Read\":47566,\"Records Read\":921},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerStageCompleted\",\"Stage Info\":{\"Stage ID\":0,\"Stage Attempt ID\":0,\"Stage Name\":\"mapToPair at JavaWordCount.java:49\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":6,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"4\\\",\\\"name\\\":\\\"map\\\"}\",\"Callsite\":\"mapToPair at JavaWordCount.java:49\",\"Parent IDs\":[5],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":0,\"Name\":\"FileScanRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":4,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"2\\\",\\\"name\\\":\\\"mapPartitions\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[3],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":2,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"0\\\",\\\"name\\\":\\\"DeserializeToObject\\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[1],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":3,\"Name\":\"SQLExecutionRDD\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[2],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":5,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"3\\\",\\\"name\\\":\\\"flatMap\\\"}\",\"Callsite\":\"flatMap at JavaWordCount.java:47\",\"Parent IDs\":[4],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0},{\"RDD ID\":1,\"Name\":\"MapPartitionsRDD\",\"Scope\":\"{\\\"id\\\":\\\"1\\\",\\\"name\\\":\\\"Scan text \\\"}\",\"Callsite\":\"javaRDD at JavaWordCount.java:45\",\"Parent IDs\":[0],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"DETERMINATE\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.mapToPair(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:49)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Submission Time\":1728301867588,\"Completion Time\":1728301873773,\"Accumulables\":[{\"ID\":0,\"Name\":\"number of output rows\",\"Value\":\"921\",\"Internal\":true,\"Count Failed Values\":true,\"Metadata\":\"sql\"},{\"ID\":4,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Value\":1159,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":5,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Value\":864336864,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":6,\"Name\":\"internal.metrics.executorRunTime\",\"Value\":4604,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":7,\"Name\":\"internal.metrics.executorCpuTime\",\"Value\":2593529632,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":8,\"Name\":\"internal.metrics.resultSize\",\"Value\":1842,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":9,\"Name\":\"internal.metrics.jvmGCTime\",\"Value\":99,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":10,\"Name\":\"internal.metrics.resultSerializationTime\",\"Value\":4,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":11,\"Name\":\"internal.metrics.memoryBytesSpilled\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":12,\"Name\":\"internal.metrics.diskBytesSpilled\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":13,\"Name\":\"internal.metrics.peakExecutionMemory\",\"Value\":219951,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":32,\"Name\":\"internal.metrics.shuffle.write.bytesWritten\",\"Value\":21712,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":33,\"Name\":\"internal.metrics.shuffle.write.recordsWritten\",\"Value\":2357,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":34,\"Name\":\"internal.metrics.shuffle.write.writeTime\",\"Value\":1716215,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":35,\"Name\":\"internal.metrics.input.bytesRead\",\"Value\":47566,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":36,\"Name\":\"internal.metrics.input.recordsRead\",\"Value\":921,\"Internal\":true,\"Count Failed Values\":true}],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0}}\n",
"{\"Event\":\"SparkListenerStageSubmitted\",\"Stage Info\":{\"Stage ID\":1,\"Stage Attempt ID\":0,\"Stage Name\":\"collect at JavaWordCount.java:53\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":7,\"Name\":\"ShuffledRDD\",\"Scope\":\"{\\\"id\\\":\\\"5\\\",\\\"name\\\":\\\"reduceByKey\\\"}\",\"Callsite\":\"reduceByKey at JavaWordCount.java:51\",\"Parent IDs\":[6],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"UNORDERED\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[0],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Submission Time\":1728301873823,\"Accumulables\":[],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0},\"Properties\":{\"spark.rdd.scope\":\"{\\\"id\\\":\\\"6\\\",\\\"name\\\":\\\"collect\\\"}\",\"spark.rdd.scope.noOverride\":\"true\"}}\n",
"{\"Event\":\"SparkListenerTaskStart\",\"Stage ID\":1,\"Stage Attempt ID\":0,\"Task Info\":{\"Task ID\":1,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301873862,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"NODE_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":0,\"Failed\":false,\"Killed\":false,\"Accumulables\":[]}}\n",
"{\"Event\":\"SparkListenerTaskEnd\",\"Stage ID\":1,\"Stage Attempt ID\":0,\"Task Type\":\"ResultTask\",\"Task End Reason\":{\"Reason\":\"Success\"},\"Task Info\":{\"Task ID\":1,\"Index\":0,\"Attempt\":0,\"Partition ID\":0,\"Launch Time\":1728301873862,\"Executor ID\":\"0\",\"Host\":\"172.28.0.12\",\"Locality\":\"NODE_LOCAL\",\"Speculative\":false,\"Getting Result Time\":0,\"Finish Time\":1728301874427,\"Failed\":false,\"Killed\":false,\"Accumulables\":[{\"ID\":39,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Update\":181,\"Value\":181,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":40,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Update\":79298899,\"Value\":79298899,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":41,\"Name\":\"internal.metrics.executorRunTime\",\"Update\":287,\"Value\":287,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":42,\"Name\":\"internal.metrics.executorCpuTime\",\"Update\":174608711,\"Value\":174608711,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":43,\"Name\":\"internal.metrics.resultSize\",\"Update\":52992,\"Value\":52992,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":44,\"Name\":\"internal.metrics.jvmGCTime\",\"Update\":24,\"Value\":24,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":45,\"Name\":\"internal.metrics.resultSerializationTime\",\"Update\":9,\"Value\":9,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":46,\"Name\":\"internal.metrics.memoryBytesSpilled\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":47,\"Name\":\"internal.metrics.diskBytesSpilled\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":48,\"Name\":\"internal.metrics.peakExecutionMemory\",\"Update\":153591,\"Value\":153591,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":50,\"Name\":\"internal.metrics.shuffle.read.remoteBlocksFetched\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":51,\"Name\":\"internal.metrics.shuffle.read.localBlocksFetched\",\"Update\":1,\"Value\":1,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":52,\"Name\":\"internal.metrics.shuffle.read.remoteBytesRead\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":53,\"Name\":\"internal.metrics.shuffle.read.remoteBytesReadToDisk\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":54,\"Name\":\"internal.metrics.shuffle.read.localBytesRead\",\"Update\":21712,\"Value\":21712,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":55,\"Name\":\"internal.metrics.shuffle.read.fetchWaitTime\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":56,\"Name\":\"internal.metrics.shuffle.read.recordsRead\",\"Update\":2357,\"Value\":2357,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":57,\"Name\":\"internal.metrics.shuffle.push.read.corruptMergedBlockChunks\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":58,\"Name\":\"internal.metrics.shuffle.push.read.mergedFetchFallbackCount\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":59,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":60,\"Name\":\"internal.metrics.shuffle.push.read.localMergedBlocksFetched\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":61,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedChunksFetched\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":62,\"Name\":\"internal.metrics.shuffle.push.read.localMergedChunksFetched\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":63,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedBytesRead\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":64,\"Name\":\"internal.metrics.shuffle.push.read.localMergedBytesRead\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":65,\"Name\":\"internal.metrics.shuffle.read.remoteReqsDuration\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":66,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedReqsDuration\",\"Update\":0,\"Value\":0,\"Internal\":true,\"Count Failed Values\":true}]},\"Task Executor Metrics\":{\"JVMHeapMemory\":0,\"JVMOffHeapMemory\":0,\"OnHeapExecutionMemory\":0,\"OffHeapExecutionMemory\":0,\"OnHeapStorageMemory\":0,\"OffHeapStorageMemory\":0,\"OnHeapUnifiedMemory\":0,\"OffHeapUnifiedMemory\":0,\"DirectPoolMemory\":0,\"MappedPoolMemory\":0,\"ProcessTreeJVMVMemory\":0,\"ProcessTreeJVMRSSMemory\":0,\"ProcessTreePythonVMemory\":0,\"ProcessTreePythonRSSMemory\":0,\"ProcessTreeOtherVMemory\":0,\"ProcessTreeOtherRSSMemory\":0,\"MinorGCCount\":0,\"MinorGCTime\":0,\"MajorGCCount\":0,\"MajorGCTime\":0,\"TotalGCTime\":0},\"Task Metrics\":{\"Executor Deserialize Time\":181,\"Executor Deserialize CPU Time\":79298899,\"Executor Run Time\":287,\"Executor CPU Time\":174608711,\"Peak Execution Memory\":153591,\"Result Size\":52992,\"JVM GC Time\":24,\"Result Serialization Time\":9,\"Memory Bytes Spilled\":0,\"Disk Bytes Spilled\":0,\"Shuffle Read Metrics\":{\"Remote Blocks Fetched\":0,\"Local Blocks Fetched\":1,\"Fetch Wait Time\":0,\"Remote Bytes Read\":0,\"Remote Bytes Read To Disk\":0,\"Local Bytes Read\":21712,\"Total Records Read\":2357,\"Remote Requests Duration\":0,\"Push Based Shuffle\":{\"Corrupt Merged Block Chunks\":0,\"Merged Fetch Fallback Count\":0,\"Merged Remote Blocks Fetched\":0,\"Merged Local Blocks Fetched\":0,\"Merged Remote Chunks Fetched\":0,\"Merged Local Chunks Fetched\":0,\"Merged Remote Bytes Read\":0,\"Merged Local Bytes Read\":0,\"Merged Remote Requests Duration\":0}},\"Shuffle Write Metrics\":{\"Shuffle Bytes Written\":0,\"Shuffle Write Time\":0,\"Shuffle Records Written\":0},\"Input Metrics\":{\"Bytes Read\":0,\"Records Read\":0},\"Output Metrics\":{\"Bytes Written\":0,\"Records Written\":0},\"Updated Blocks\":[]}}\n",
"{\"Event\":\"SparkListenerStageCompleted\",\"Stage Info\":{\"Stage ID\":1,\"Stage Attempt ID\":0,\"Stage Name\":\"collect at JavaWordCount.java:53\",\"Number of Tasks\":1,\"RDD Info\":[{\"RDD ID\":7,\"Name\":\"ShuffledRDD\",\"Scope\":\"{\\\"id\\\":\\\"5\\\",\\\"name\\\":\\\"reduceByKey\\\"}\",\"Callsite\":\"reduceByKey at JavaWordCount.java:51\",\"Parent IDs\":[6],\"Storage Level\":{\"Use Disk\":false,\"Use Memory\":false,\"Use Off Heap\":false,\"Deserialized\":false,\"Replication\":1},\"Barrier\":false,\"DeterministicLevel\":\"UNORDERED\",\"Number of Partitions\":1,\"Number of Cached Partitions\":0,\"Memory Size\":0,\"Disk Size\":0}],\"Parent IDs\":[0],\"Details\":\"org.apache.spark.api.java.AbstractJavaRDDLike.collect(JavaRDDLike.scala:45)\\norg.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:53)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\\njava.base/java.lang.reflect.Method.invoke(Method.java:566)\\norg.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)\\norg.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1029)\\norg.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:194)\\norg.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:217)\\norg.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:91)\\norg.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1120)\\norg.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1129)\\norg.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)\",\"Submission Time\":1728301873823,\"Completion Time\":1728301874430,\"Accumulables\":[{\"ID\":39,\"Name\":\"internal.metrics.executorDeserializeTime\",\"Value\":181,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":40,\"Name\":\"internal.metrics.executorDeserializeCpuTime\",\"Value\":79298899,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":41,\"Name\":\"internal.metrics.executorRunTime\",\"Value\":287,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":42,\"Name\":\"internal.metrics.executorCpuTime\",\"Value\":174608711,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":43,\"Name\":\"internal.metrics.resultSize\",\"Value\":52992,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":44,\"Name\":\"internal.metrics.jvmGCTime\",\"Value\":24,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":45,\"Name\":\"internal.metrics.resultSerializationTime\",\"Value\":9,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":46,\"Name\":\"internal.metrics.memoryBytesSpilled\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":47,\"Name\":\"internal.metrics.diskBytesSpilled\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":48,\"Name\":\"internal.metrics.peakExecutionMemory\",\"Value\":153591,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":50,\"Name\":\"internal.metrics.shuffle.read.remoteBlocksFetched\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":51,\"Name\":\"internal.metrics.shuffle.read.localBlocksFetched\",\"Value\":1,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":52,\"Name\":\"internal.metrics.shuffle.read.remoteBytesRead\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":53,\"Name\":\"internal.metrics.shuffle.read.remoteBytesReadToDisk\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":54,\"Name\":\"internal.metrics.shuffle.read.localBytesRead\",\"Value\":21712,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":55,\"Name\":\"internal.metrics.shuffle.read.fetchWaitTime\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":56,\"Name\":\"internal.metrics.shuffle.read.recordsRead\",\"Value\":2357,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":57,\"Name\":\"internal.metrics.shuffle.push.read.corruptMergedBlockChunks\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":58,\"Name\":\"internal.metrics.shuffle.push.read.mergedFetchFallbackCount\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":59,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":60,\"Name\":\"internal.metrics.shuffle.push.read.localMergedBlocksFetched\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":61,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedChunksFetched\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":62,\"Name\":\"internal.metrics.shuffle.push.read.localMergedChunksFetched\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":63,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedBytesRead\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":64,\"Name\":\"internal.metrics.shuffle.push.read.localMergedBytesRead\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":65,\"Name\":\"internal.metrics.shuffle.read.remoteReqsDuration\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true},{\"ID\":66,\"Name\":\"internal.metrics.shuffle.push.read.remoteMergedReqsDuration\",\"Value\":0,\"Internal\":true,\"Count Failed Values\":true}],\"Resource Profile Id\":0,\"Shuffle Push Enabled\":false,\"Shuffle Push Mergers Count\":0}}\n",
"{\"Event\":\"SparkListenerJobEnd\",\"Job ID\":0,\"Completion Time\":1728301874450,\"Job Result\":{\"Result\":\"JobSucceeded\"}}\n",
"{\"Event\":\"SparkListenerApplicationEnd\",\"Timestamp\":1728301874564}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"### Spark configuration\n",
"\n",
"To customize Spark use the configuration templates in `$SPARK_HOME/conf` (remove the template extension).\n"
],
"metadata": {
"id": "S8LCE08ZtBHv"
}
},
{
"cell_type": "code",
"source": [
"!ls -al $SPARK_HOME/conf"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "mFYEnnrOw179",
"outputId": "651b7ebd-4c08-4699-9b36-5c9242c28c5e"
},
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"total 44\n",
"drwxr-xr-x 2 1000 1000 4096 Sep 9 05:33 .\n",
"drwxr-xr-x 15 1000 1000 4096 Oct 7 11:50 ..\n",
"-rw-r--r-- 1 1000 1000 1105 Sep 9 05:33 fairscheduler.xml.template\n",
"-rw-r--r-- 1 1000 1000 3350 Sep 9 05:33 log4j2.properties.template\n",
"-rw-r--r-- 1 1000 1000 9141 Sep 9 05:33 metrics.properties.template\n",
"-rw-r--r-- 1 1000 1000 1292 Sep 9 05:33 spark-defaults.conf.template\n",
"-rwxr-xr-x 1 1000 1000 4694 Sep 9 05:33 spark-env.sh.template\n",
"-rw-r--r-- 1 1000 1000 865 Sep 9 05:33 workers.template\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## Shutdown\n",
"\n",
"Stop all services."
],
"metadata": {
"id": "AZuz6m48UyoO"
}
},
{
"cell_type": "code",
"source": [
"%%bash\n",
"$SPARK_HOME/sbin/stop-history-server.sh\n",
"$SPARK_HOME/sbin/stop-worker.sh\n",
"$SPARK_HOME/sbin/stop-master.sh"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "KE4B2RY5U4wG",
"outputId": "e6605ccb-1e0e-41a1-c0c2-bd190806aa4e"
},
"execution_count": 15,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"stopping org.apache.spark.deploy.history.HistoryServer\n",
"stopping org.apache.spark.deploy.worker.Worker\n",
"stopping org.apache.spark.deploy.master.Master\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"Terminate the ngrok processes."
],
"metadata": {
"id": "ckDIFrNMcSGf"
}
},
{
"cell_type": "code",
"source": [
"if NGROK:\n",
" ngrok.kill()"
],
"metadata": {
"id": "DLGmZykeb3UW"
},
"execution_count": 16,
"outputs": []
}
]
}![\"Logo](\"https://github.com/groda/big_data/blob/master/logo_bdb.png?raw=true\"){kind=logo}