don't add optional dependencies to requirements.txt

Signed-off-by: Vikrant Puppala <vikrant.puppala@databricks.com>

Author: vikrantpuppala

scripts/dependency_manager.py

@@ -27,8 +27,11 @@ def _parse_constraint(self, name, constraint):
         if isinstance(constraint, str):
             return constraint, False  # version_constraint, is_optional
         elif isinstance(constraint, list):
-            # Handle complex constraints like pandas
-            return constraint[0]['version'], False
+            # Handle complex constraints like pandas/pyarrow
+            first_constraint = constraint[0]
+            version = first_constraint['version']
+            is_optional = first_constraint.get('optional', False)
+            return version, is_optional
         elif isinstance(constraint, dict):
             if 'version' in constraint:
                 return constraint['version'], constraint.get('optional', False)