import luigi from pyspark.sql import SQLContext from luigi.contrib.spark import PySparkTask, SparkSubmitTask import logging from helloworld_task import HelloWorldTask class PySparkTableSchema(PySparkTask): def requires(self): return HelloWorldTask() def output(self): return luigi.LocalTarget('pysparkout.csv') def main(self, sc, *args): logging.info("=======SPARK JOB=========") sqlContext = SQLContext(sc) df = (sqlContext.load(source="jdbc", url="jdbc:postgresql://localhost:2222/mydatabase?user=dbuser&password=dbpassword", dbtable="tablename")) print(df.show()) logging.info(df.printSchema()) with self.output().open('w') as outFile: outFile.write(str(result))