(ns)
A generic plugin that uses Spark to:
        read data,
        transform data with given code,
        and maybe write transformed data

    Assume code is written in Python.  For Scala or R code, use another option.
A generic plugin that uses Spark to:
        read data,
        transform data with given code,
        and maybe write transformed data
A generic plugin that uses Spark to : read data transform data with given code and maybe write transformed data
def main(ns):
    """
    A generic plugin that uses Spark to:
        read data,
        transform data with given code,
        and maybe write transformed data

    Assume code is written in Python.  For Scala or R code, use another option.
    """
    job_id = ns.job_id
    module = get_pymodule(ns.app_name)

    pjob_id = api.parse_job_id(ns.app_name, job_id)
    log_details = dict(
        module_name=module.__name__,
        app_name=ns.app_name, job_id=job_id)

    conf, osenv, files, pyFiles = pyspark_context.get_spark_conf(ns.app_name)
    conf['spark.app.name'] = "%s__%s" % (conf['spark.app.name'], job_id)
    conf.update(ns.spark_conf)
    osenv.update(ns.spark_env)
    sc = pyspark_context.get_spark_context(
        conf=conf, osenv=osenv, files=files, pyFiles=pyFiles)
    apply_data_transform(
        ns=ns, sc=sc, log_details=log_details, pjob_id=pjob_id, module=module)
    sc.stop()
def main ( ns ) : job_id = ns . job_id module = get_pymodule ( ns . app_name ) pjob_id = api . parse_job_id ( ns . app_name , job_id ) log_details = dict ( module_name = module . __name__ , app_name = ns . app_name , job_id = job_id ) conf , osenv , files , pyFiles = pyspark_context . get_spark_conf ( ns . app_name ) conf [ 'spark.app.name' ] = "%s__%s" % ( conf [ 'spark.app.name' ] , job_id ) conf . update ( ns . spark_conf ) osenv . update ( ns . spark_env ) sc = pyspark_context . get_spark_context ( conf = conf , osenv = osenv , files = files , pyFiles = pyFiles ) apply_data_transform ( ns = ns , sc = sc , log_details = log_details , pjob_id = pjob_id , module = module ) sc . stop ( )