log

2024-03-18 09:46:38,889 WARN util.Utils: spark.executor.instances less than spark.dynamicAllocation.minExecutors is invalid, ignoring its setting, please update your configs.
2024-03-18 09:46:43,266 WARN util.Utils: spark.executor.instances less than spark.dynamicAllocation.minExecutors is invalid, ignoring its setting, please update your configs.
2024-03-18 09:46:43,267 WARN cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!
---------------------------------------------------------------------------
Py4JError                                 Traceback (most recent call last)
Cell In[11], line 1
----> 1 spark = SparkSession.builder.master("yarn").appName("L5T2").getOrCreate()

File /usr/local/lib/python3.8/dist-packages/pyspark/sql/session.py:497, in SparkSession.Builder.getOrCreate(self)
    495     sparkConf.set(key, value)
    496 # This SparkContext may be an existing one.
--> 497 sc = SparkContext.getOrCreate(sparkConf)
    498 # Do not update `SparkConf` for existing `SparkContext`, as it's shared
    499 # by all sessions.
    500 session = SparkSession(sc, options=self._options)

File /usr/local/lib/python3.8/dist-packages/pyspark/context.py:515, in SparkContext.getOrCreate(cls, conf)
    513 with SparkContext._lock:
    514     if SparkContext._active_spark_context is None:
--> 515         SparkContext(conf=conf or SparkConf())
    516     assert SparkContext._active_spark_context is not None
    517     return SparkContext._active_spark_context

File /usr/local/lib/python3.8/dist-packages/pyspark/context.py:203, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls, udf_profiler_cls, memory_profiler_cls)
    201 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
    202 try:
--> 203     self._do_init(
    204         master,
    205         appName,
    206         sparkHome,
    207         pyFiles,
    208         environment,
    209         batchSize,
    210         serializer,
    211         conf,
    212         jsc,
    213         profiler_cls,
    214         udf_profiler_cls,
    215         memory_profiler_cls,
    216     )
    217 except BaseException:
    218     # If an error occurs, clean up in order to allow future SparkContext creation:
    219     self.stop()

File /usr/local/lib/python3.8/dist-packages/pyspark/context.py:316, in SparkContext._do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls, udf_profiler_cls, memory_profiler_cls)
    311 # If encryption is enabled, we need to setup a server in the jvm to read broadcast
    312 # data via a socket.
    313 # scala's mangled names w/ $ in them require special treatment.
    314 self._encryption_enabled = self._jvm.PythonUtils.isEncryptionEnabled(self._jsc)
    315 os.environ["SPARK_AUTH_SOCKET_TIMEOUT"] = str(
--> 316     self._jvm.PythonUtils.getPythonAuthSocketTimeout(self._jsc)
    317 )
    318 os.environ["SPARK_BUFFER_SIZE"] = str(self._jvm.PythonUtils.getSparkBufferSize(self._jsc))
    320 self.pythonExec = os.environ.get("PYSPARK_PYTHON", "python3")

File /usr/local/lib/python3.8/dist-packages/py4j/java_gateway.py:1549, in JavaClass.__getattr__(self, name)
   1546         return get_return_value(
   1547             answer, self._gateway_client, self._fqn, name)
   1548 else:
-> 1549     raise Py4JError(
   1550         "{0}.{1} does not exist in the JVM".format(self._fqn, name))

Py4JError: org.apache.spark.api.python.PythonUtils.getPythonAuthSocketTimeout does not exist in the JVM