resolve conflicts for aws-samples#13

Author: manabery

utilities/Hive_metastore_migration/README.md

@@ -206,8 +206,17 @@ as an Glue ETL job, if AWS Glue can directly connect to your Hive metastore.
    - `--database-prefix` and `--table-prefix` (optional) to set a string prefix that is applied to the 
      database and table names. They are empty by default. 
 
+   - Optionally, you can set `--config_file` to `<path_to_your_config_yaml_file>` which contains configuration parameters (default path: `artifacts/config.yaml`)
+     - Provide the following configuration parameters in the configuration yaml file:
+        * mode
+        * jdbc-url
+        * jdbc-username
+        * jdbc-password
+        * database-prefix
+        * table-prefix
+
    - Example spark-submit command to migrate Hive metastore to S3, tested on EMR-4.7.1:
-    ```bash
+   ```bash
     MYSQL_JAR_PATH=/usr/lib/hadoop/mysql-connector-java-5.1.42-bin.jar
     DRIVER_CLASSPATH=/home/hadoop/*:/etc/hadoop/conf:/etc/hive/conf:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/emrfs/auxlib/*:$MYSQL_JAR_PATH
     spark-submit --driver-class-path $DRIVER_CLASSPATH \

utilities/Hive_metastore_migration/artifacts/config.yaml

@@ -0,0 +1,8 @@
+mode:
+jdbc-url:
+jdbc-username:
+jdbc-password:
+database-prefix:
+table-prefix:
+output-path:
+input_path:

utilities/Hive_metastore_migration/src/hive_metastore_migration.py

@@ -9,6 +9,7 @@
 # except for python 2.7 standard library and Spark 2.1
 import sys
 from datetime import datetime, timedelta, tzinfo
+import yaml
 from time import localtime, strftime
 from types import MethodType
 
@@ -1606,6 +1607,39 @@ def parse_arguments(args):
     return options
 
 
+def parse_arguments_from_yaml_file(args):
+    """
+    This function accepts the path to a config file
+    and extracts the needed arguments for the metastore migration
+    ----------
+    Return:
+        Dictionary of config options
+    """
+    parser = argparse.ArgumentParser(prog=args[0])
+    parser.add_argument('-f', '--config_file', required=True, default='artifacts/config.yaml`', help='Provide yaml configuration file path to read migration arguments from. Default path: `artifacts/config.yaml`')
+    options = get_options(parser, args)
+    config_file_path = options['config_file']
+    ## read the yaml file
+    with open(config_file_path, 'r') as yaml_file_stream:
+        config_options = yaml.load(yaml_file_stream)
+
+    if config_options['mode'] == FROM_METASTORE:
+        validate_options_in_mode(
+            options=config_options, mode=FROM_METASTORE,
+            required_options=['output_path'],
+            not_allowed_options=['input_path']
+        )
+    elif config_options['mode'] == TO_METASTORE:
+        validate_options_in_mode(
+            options=config_options, mode=TO_METASTORE,
+            required_options=['input_path'],
+            not_allowed_options=['output_path']
+        )
+    else:
+        raise AssertionError('unknown mode ' + options['mode'])
+
+    return config_options
+
 def get_spark_env():
     try:
         sc = SparkContext.getOrCreate()
@@ -1733,7 +1767,10 @@ def validate_aws_regions(region):
 
 
 def main():
-    options = parse_arguments(sys.argv)
+    # options = parse_arguments(sys.argv)
+
+    ## This now reads options from path to config yaml file
+    options = parse_arguments_from_yaml_file(sys.argv)
 
     connection = {"url": options["jdbc_url"], "user": options["jdbc_username"], "password": options["jdbc_password"]}
     db_prefix = options.get("database_prefix") or ""