code: def run_pyspark_yarn_cluster(env_dir, env_name, env_archive, args):
      
      env = dict(os.environ)
      yarn_python = os.path.join(".", "CONDA", env_name, "bin", "python")
      archives = env_archive + "#CONDA"
  
      prepend_args = [
          "--master", "yarn",
          "--deploy-mode", "cluster",
          "--conf", "spark.yarn.appMasterEnv.PYSPARK_PYTHON={}".format(yarn_python),
          "--archives", archives,
      ]
  
      env_update = {
          "PYSPARK_PYTHON": yarn_python
      }
  
      env.update(env_update)
      spark_submit = os.path.join(env["SPARK_HOME"], "bin", "spark-submit")
  
      log.info("Running spark in YARN-client mode with added arguments")
      log.info("  args: %s", pprint.pprint(prepend_args, indent=4))
      log.info("  env: %s", pprint.pprint(env_update, indent=4))
      # REPLACE our python process with another one
      subprocess.check_call([spark_submit] + prepend_args + args, env=env)
docs: Initializes the requires spark command line options on order to start a python job with the given python environment.
  
      Parameters
      ----------
      env_dir : str
      env_name : str
      env_archive : str
      args : list
  
      Returns
      -------
      This call will spawn a child process and block until that is complete.