Merge branch 'main' into feature/declare-variable-support

Author: KeilorRodriguez

docs/lakebridge/docs/installation.mdx

@@ -112,6 +112,20 @@ databricks labs lakebridge install-transpile
 
 <img src={useBaseUrl('img/transpile-install.gif')} alt="transpile-install" />
 
+
+:::tip
+#### Override the default[Bladebridge] config:
+
+There is an option for you to override the default config file that `Lakebridge` uses for converting source code from dialects like `informatica`, `datastage`, `synapse`,
+`orcale` etc. During installtion you may use your own custom config file and `Lakebridge` will override the config with the one you would provide. You can only setup this
+override during installation.
+
+Specify the config file to override the default[Bladebridge] config during installation:
+```bash
+Specify the config file to override the default[Bladebridge] config - press <enter> for none (default: <none>): <local_full_path>/custom_<source>2databricks.json
+```
+:::
+
 ### Verify Installation
 Verify the successful installation by executing the provided command; confirmation of a successful installation is indicated when the displayed output aligns with the example screenshot provided:
 ```bash
@@ -136,7 +150,7 @@ databricks labs lakebridge configure-reconcile
 ### SQL Warehouse for Reconcile
 
 While configuring the reconcile properties, lakebridge by default creates a SQL warehouse. lakebridge uses user profile to authenticate to any Databricks resource and hence
-if the user running this command doesn't have permission to create SQL warehouse, the configure-reconcile would fail. In this case users can provide the 
+if the user running this command doesn't have permission to create SQL warehouse, the configure-reconcile would fail. In this case users can provide the
 `warehouse_id` of an already created SQL warehouse that they have atleast CAN_USE permission on in the databricks profile (`~/.databrickscfg`) using which they
 are running the lakebridge commands and lakebridge would use that warehouse to complete the reconcile configuration instead of trying to create a new one.
 

docs/lakebridge/docs/transpile/index.mdx

@@ -20,10 +20,10 @@ choose to configure this at the time of installation, or choose defaults/set lat
 transpile call.
 
 The `transpile` command will trigger the conversion of the specified code, these are the available arguments:
-- `source-dialect [Required]` - Dialect name (ex: snowflake, oracle, datastage, etc)
 - `input-source [Required]` - The path to the SQL file or directory containing SQL files to be transpiled.
 - `output-folder [Optional]` - The path to the output folder where the transpiled SQL files will be stored. If not specified,
 the transpiled SQL files will be stored in a folder called `transpiled` in your current working directory.
+- `source-dialect [Optional]` - Dialect name (ex: snowflake, oracle, datastage, etc). If not specified, refers to the Source Dialect selected at installation time.
 - `error-file-path [Optional]` - The path to the file where the transpile errors will be stored. If not specified, the
 errors will be stored in a file called `errors.log` in your current working directory.
 - `skip-validation [Optional]` - The default value is True. If set to False, the transpiler will validate the transpiled SQL scripts against the Databricks catalog and schema provided by user.

docs/lakebridge/package.json

@@ -59,6 +59,7 @@
   },
   "resolutions": {
     "brace-expansion": "^1.1.12",
-    "webpack-dev-server": "^5.2.1"
+    "webpack-dev-server": "^5.2.1",
+    "on-headers": ">=1.1.0"
   }
 }

docs/lakebridge/yarn.lock

@@ -7509,10 +7509,10 @@ on-finished@2.4.1, on-finished@^2.4.1:
   dependencies:
     ee-first "1.1.1"
 
-on-headers@~1.0.2:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.2.tgz#772b0ae6aaa525c399e489adfad90c403eb3c28f"
-  integrity sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==
+on-headers@>=1.1.0, on-headers@~1.0.2:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.1.0.tgz#59da4f91c45f5f989c6e4bcedc5a3b0aed70ff65"
+  integrity sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==
 
 onetime@^5.1.2:
   version "5.1.2"

labs.yml

@@ -22,19 +22,19 @@ commands:
     description: Transpile SQL script to Databricks SQL
     flags:
       - name: transpiler-config-path
-        description: Path to the transpiler configuration file
+        description: Path to the transpiler configuration file [Pluggable Transpilers] eg:- Morpheus/Bladebridge
         default: null
       - name: source-dialect
-        description: Dialect name
+        description: Dialect name as selected during `install-transpile` or refer to documentation
         default: null
       - name: input-source
         description: Input Script Folder or File
         default: null
       - name: output-folder
-        description: Output Location For Storing Transpiled Code, defaults to input-source folder
+        description: Output Location For Storing Transpiled Code, defaults to (Current Working Directory) `<pwd>/transpiled` folder
         default: null
       - name: error-file-path
-        description: Output Location For Storing Errors, defaults to input-source folder
+        description: Output Location For Storing Errors, defaults to (Current Working Directory) `<pwd>/errors.log`
         default: null
       - name: skip-validation
         description: Validate Transpiled Code, default True validation skipped, False validate

tests/resources/transpiler_configs/bladebridge/lib/config.yml

@@ -19,7 +19,7 @@ options:
   all:
     - flag: "overrides-file"
       method: QUESTION
-      prompt: "Specify the file to use for overrides - press <enter> for none"
+      prompt: "Specify the config file to override the default[Bladebridge] config - press <enter> for none"
       default: "<none>"
   datastage:
     - flag: "target-tech"

tests/unit/transpiler/test_sqlglot_engine.py

@@ -114,7 +114,6 @@ def test_safe_parse(transpiler, transpile_config):
     )
     for exp in result:
         if exp.parsed_expression:
-            print("yes")
             assert repr(exp.parsed_expression.args["expressions"]) == repr(expected_result)
             assert repr(exp.parsed_expression.args["from"]) == repr(expected_from_result)
     assert error[0].transpile_error.code == "PARSE_ERROR"