
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
from pyspark.sql import SparkSession


def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('root_dir')
  parser.add_argument('tables', type=lambda csv: csv.split(','))
  args = parser.parse_args()
  spark = (SparkSession.builder
           .appName('Setup Spark tables')
           .enableHiveSupport()
           .getOrCreate())
  for table in args.tables:
    logging.info('Creating table %s', table)
    table_dir = os.path.join(args.root_dir, table)
    # clean up previous table
    spark.sql('drop table if exists ' + table)
    # register new table
    spark.catalog.createTable(table, table_dir, source='parquet')

if __name__ == '__main__':
  main()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
from pyspark.sql import SparkSession
from pyspark.sql.utils import AnalysisException


def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('root_dir')
  parser.add_argument('tables', type=lambda csv: csv.split(','))
  args = parser.parse_args()
  spark = (SparkSession.builder
           .appName('Setup Spark tables')
           .enableHiveSupport()
           .getOrCreate())
  for table in args.tables:
    logging.info('Creating table %s', table)
    table_dir = os.path.join(args.root_dir, table)
    # clean up previous table
    spark.sql('DROP TABLE IF EXISTS ' + table)
    # register new table
    spark.catalog.createTable(table, table_dir, source='parquet')
    try:
      # This loads the partitions under the table if table is partitioned.
      spark.sql('MSCK REPAIR TABLE ' + table)
    except AnalysisException:
      # The table was not partitioned, which was presumably expected
      pass
if __name__ == '__main__':
  main()

Support creating Hive tables with partitioned data.
Support creating Hive tables with partitioned data.

PiperOrigin-RevId: 335539022

  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  import argparse
  import logging
  import os
  from pyspark.sql import SparkSession
+ from pyspark.sql.utils import AnalysisException
  
  
  def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('root_dir')
    parser.add_argument('tables', type=lambda csv: csv.split(','))
    args = parser.parse_args()
    spark = (SparkSession.builder
             .appName('Setup Spark tables')
             .enableHiveSupport()
             .getOrCreate())
    for table in args.tables:
      logging.info('Creating table %s', table)
      table_dir = os.path.join(args.root_dir, table)
      # clean up previous table
-     spark.sql('drop table if exists ' + table)
+     spark.sql('DROP TABLE IF EXISTS ' + table)
      # register new table
      spark.catalog.createTable(table, table_dir, source='parquet')
- 
+     try:
+       # This loads the partitions under the table if table is partitioned.
+       spark.sql('MSCK REPAIR TABLE ' + table)
+     except AnalysisException:
+       # The table was not partitioned, which was presumably expected
+       pass
  if __name__ == '__main__':
    main()
  
Support creating Hive tables with partitioned data.
## Code Before:

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
from pyspark.sql import SparkSession


def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('root_dir')
  parser.add_argument('tables', type=lambda csv: csv.split(','))
  args = parser.parse_args()
  spark = (SparkSession.builder
           .appName('Setup Spark tables')
           .enableHiveSupport()
           .getOrCreate())
  for table in args.tables:
    logging.info('Creating table %s', table)
    table_dir = os.path.join(args.root_dir, table)
    # clean up previous table
    spark.sql('drop table if exists ' + table)
    # register new table
    spark.catalog.createTable(table, table_dir, source='parquet')

if __name__ == '__main__':
  main()

## Instruction:
Support creating Hive tables with partitioned data.
## Code After:

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import logging
import os
from pyspark.sql import SparkSession
from pyspark.sql.utils import AnalysisException


def main():
  parser = argparse.ArgumentParser()
  parser.add_argument('root_dir')
  parser.add_argument('tables', type=lambda csv: csv.split(','))
  args = parser.parse_args()
  spark = (SparkSession.builder
           .appName('Setup Spark tables')
           .enableHiveSupport()
           .getOrCreate())
  for table in args.tables:
    logging.info('Creating table %s', table)
    table_dir = os.path.join(args.root_dir, table)
    # clean up previous table
    spark.sql('DROP TABLE IF EXISTS ' + table)
    # register new table
    spark.catalog.createTable(table, table_dir, source='parquet')
    try:
      # This loads the partitions under the table if table is partitioned.
      spark.sql('MSCK REPAIR TABLE ' + table)
    except AnalysisException:
      # The table was not partitioned, which was presumably expected
      pass
if __name__ == '__main__':
  main()
