From f8b36439f527d51cd6e3b237e13f6b380f38ce6f Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 01:22:24 +0900 Subject: [PATCH 01/11] get dockerfile by calling api --- core/morph/task/deploy.py | 54 +++++++++++------------- core/morph/task/utils/load_dockerfile.py | 39 +++++++++++++++++ 2 files changed, 63 insertions(+), 30 deletions(-) create mode 100644 core/morph/task/utils/load_dockerfile.py diff --git a/core/morph/task/deploy.py b/core/morph/task/deploy.py index 629c0e3..8884b3a 100644 --- a/core/morph/task/deploy.py +++ b/core/morph/task/deploy.py @@ -8,14 +8,16 @@ import click import requests +from tqdm import tqdm + from morph.api.cloud.client import MorphApiKeyClientImpl from morph.api.cloud.types import EnvVarObject from morph.cli.flags import Flags from morph.config.project import load_project from morph.task.base import BaseTask from morph.task.utils.file_upload import FileWithProgress +from morph.task.utils.load_dockerfile import get_dockerfile_from_api from morph.task.utils.morph import find_project_root_dir -from tqdm import tqdm class DeployTask(BaseTask): @@ -49,9 +51,24 @@ def __init__(self, args: Flags): # Check Dockerfile existence self.dockerfile = os.path.join(self.project_root, "Dockerfile") - if not os.path.exists(self.dockerfile): - click.echo(click.style(f"Error: {self.dockerfile} not found", fg="red")) - sys.exit(1) + if self.project.build is not None and self.project.build.use_custom_dockerfile: + if not os.path.exists(self.dockerfile): + click.echo(click.style(f"Error: {self.dockerfile} not found", fg="red")) + sys.exit(1) + else: + if self.project.build is None: + dockerfile, dockerignore = get_dockerfile_from_api("morph", None, None) + else: + dockerfile, dockerignore = get_dockerfile_from_api( + self.project.build.framework, + self.project.build.package_manager, + self.project.build.runtime, + ) + with open(self.dockerfile, "w") as f: + f.write(dockerfile) + dockerignore_path = os.path.join(self.project_root, ".dockerignore") + with open(dockerignore_path, "w") as f: + f.write(dockerignore) # Check Docker availability try: @@ -105,7 +122,7 @@ def run(self): click.echo(click.style("Initiating deployment sequence...", fg="blue")) # 1. Build the source code - self._copy_and_build_source() + self._build_source() # 2. Build the Docker image click.echo(click.style("Building Docker image...", fg="blue")) @@ -387,31 +404,8 @@ def _validate_api_key(self): ) sys.exit(1) - def _copy_and_build_source(self): - click.echo(click.style("Building frontend...", fg="blue")) - try: - # Run npm install and build - subprocess.run( - ["npm", "install"], - cwd=self.project_root, - check=True, - shell=True if sys.platform == "win32" else False, - ) - subprocess.run( - ["npm", "run", "build"], - cwd=self.project_root, - check=True, - shell=True if sys.platform == "win32" else False, - ) - - except subprocess.CalledProcessError as e: - click.echo(click.style(f"Error building frontend: {str(e)}", fg="red")) - sys.exit(1) - except Exception as e: - click.echo(click.style(f"Unexpected error: {str(e)}", fg="red")) - sys.exit(1) - - click.echo(click.style("Building backend...", fg="blue")) + def _build_source(self): + click.echo(click.style("Compiling morph project...", fg="blue")) try: # Compile the morph project subprocess.run( diff --git a/core/morph/task/utils/load_dockerfile.py b/core/morph/task/utils/load_dockerfile.py new file mode 100644 index 0000000..64c950f --- /dev/null +++ b/core/morph/task/utils/load_dockerfile.py @@ -0,0 +1,39 @@ +from typing import Any, Dict, Optional, Tuple + +import requests + + +def get_dockerfile_from_api( + framework: Optional[str] = "morph", + package_manager: Optional[str] = None, + runtime: Optional[str] = None, +) -> Tuple[str, str]: + """ + Fetch dockerfile and dockerignore from the Morph API. + + Args: + framework: The framework to get the dockerfile for + package_manager: Optional package manager to use + language_version: Optional language version to use + + Returns: + Tuple containing (dockerfile, dockerignore) + """ + url = f"https://backend-api-public.morph-cb9.workers.dev/dockerfile/{framework}" + + params: Dict[str, Any] = {} + if package_manager: + params["packageManager"] = package_manager + if runtime: + params["runtime"] = runtime + + response = requests.get(url, params=params) + + response.raise_for_status() + + data = response.json() + + if "error" in data: + raise ValueError(data["error"]) + + return data["dockerfile"], data["dockerignore"] From 5e699bf6db5de62aabb98385ec2c6c8678485190 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 10:36:12 +0900 Subject: [PATCH 02/11] update version as v0.3.0rc1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ccca382..9c1bd08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "morph-data" -version = "0.2.1" +version = "0.3.0rc1" description = "Morph is a python-centric full-stack framework for building and deploying data apps." authors = ["Morph "] packages = [ From b1117124063e72c123ebddf7e58298185c4de356 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 16:16:13 +0900 Subject: [PATCH 03/11] fix initial morph_project.yml --- core/morph/config/project.py | 120 ++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index f977565..ec5bf8f 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -89,9 +89,125 @@ def save_project(project_root: str, project: MorphProject) -> None: old_config_path = os.path.join(project_root, "morph_project.yaml") if os.path.exists(old_config_path): with open(old_config_path, "w") as f: - yaml.safe_dump(project.model_dump(), f) + f.write(dump_project_yaml(project)) return config_path = os.path.join(project_root, "morph_project.yml") with open(config_path, "w") as f: - yaml.safe_dump(project.model_dump(), f) + f.write(dump_project_yaml(project)) + + +def dump_project_yaml(project: MorphProject) -> str: + source_paths = "\n- ".join([""] + project.source_paths) + + # Default values + build_use_custom_dockerfile = "false" + build_runtime = "" + build_framework = "" + build_package_manager = "" + build_context = "# context: ." + build_args_str = "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" + deployment_provider = "aws" + deployment_aws_region = "us-east-1" + deployment_aws_memory = "1024" + deployment_aws_timeout = "300" + deployment_aws_concurrency = "1" + deployment_gcp_region = "us-central1" + deployment_gcp_memory = "1Gi" + deployment_gcp_cpu = "1" + deployment_gcp_concurrency = "80" + deployment_gcp_timeout = "300" + + # Set values if build exists + if project.build: + if project.build.use_custom_dockerfile is not None: + build_use_custom_dockerfile = str(project.build.use_custom_dockerfile) + if project.build.runtime: + build_runtime = project.build.runtime or "" + if project.build.framework: + build_framework = project.build.framework or "" + if project.build.package_manager: + build_package_manager = project.build.package_manager or "" + if project.build.context: + build_context = project.build.context or "# context: ." + if project.build.build_args: + build_args_items = [] + for key, value in project.build.build_args.items(): + build_args_items.append(f"{key}={value}") + build_args_str = ( + "\n - ".join([""] + build_args_items) + if build_args_items + else "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" + ) + else: + # Use default BuildConfig + build_use_custom_dockerfile = "false" + build_runtime = "" + build_framework = "" + build_package_manager = "" + build_context = "# context: ." + build_args_str = "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" + + # Set values if deployment exists + if project.deployment: + if project.deployment.provider: + deployment_provider = project.deployment.provider or "aws" + if project.deployment.aws: + deployment_aws_region = project.deployment.aws.get("region") or "us-east-1" + deployment_aws_memory = project.deployment.aws.get("memory") or "1024" + deployment_aws_timeout = project.deployment.aws.get("timeout") or "300" + deployment_aws_concurrency = ( + project.deployment.aws.get("concurrency") or "1" + ) + if project.deployment.gcp: + deployment_gcp_region = ( + project.deployment.gcp.get("region") or "us-central1" + ) + deployment_gcp_memory = project.deployment.gcp.get("memory") or "1Gi" + deployment_gcp_cpu = project.deployment.gcp.get("cpu") or "1" + deployment_gcp_concurrency = ( + project.deployment.gcp.get("concurrency") or "80" + ) + deployment_gcp_timeout = project.deployment.gcp.get("timeout") or "300" + else: + # Use default DeploymentConfig + deployment_provider = "aws" + + return f""" +# Cloud Settings +profile: {project.profile} # Defined in the Profile Section in `~/.morph/credentials` +project_id: {project.project_id} + +# Framework Settings +default_connection: {project.default_connection} +source_paths:{source_paths} + +# Build Settings +build: + use_custom_dockerfile: {build_use_custom_dockerfile} + # These settings are required when use_custom_dockerfile is false + # They define the environment in which the project will be built + runtime: {build_runtime} # python3.9, python3.10, python3.11, python3.12 + framework: {build_framework} + package_manager: {build_package_manager} # pip, poetry, uv + # These settings are required when use_custom_dockerfile is true + # They define how the Docker image will be built + context: {build_context if build_context != "# context: ." else "# context: ."} + build_args:{build_args_str} + +# Deployment Settings +deployment: + provider: {deployment_provider} # aws or gcp (default is aws) + # These settings are used only when you want to customize the deployment settings + # aws: + # region: {deployment_aws_region} + # memory: {deployment_aws_memory} + # timeout: {deployment_aws_timeout} + # concurrency: {deployment_aws_concurrency} + # gcp: + # region: {deployment_gcp_region} + # memory: {deployment_gcp_memory} + # cpu: {deployment_gcp_cpu} + # concurrency: {deployment_gcp_concurrency} + # timeout: {deployment_gcp_timeout} +""" From 54bdda572627054972476720edd0c53377701d26 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 16:31:15 +0900 Subject: [PATCH 04/11] fix initial morph_project.yml --- core/morph/config/project.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index ec5bf8f..b873853 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -105,7 +105,7 @@ def dump_project_yaml(project: MorphProject) -> str: build_runtime = "" build_framework = "" build_package_manager = "" - build_context = "# context: ." + build_context = "." build_args_str = "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" deployment_provider = "aws" deployment_aws_region = "us-east-1" @@ -121,7 +121,9 @@ def dump_project_yaml(project: MorphProject) -> str: # Set values if build exists if project.build: if project.build.use_custom_dockerfile is not None: - build_use_custom_dockerfile = str(project.build.use_custom_dockerfile) + build_use_custom_dockerfile = str( + project.build.use_custom_dockerfile + ).lower() if project.build.runtime: build_runtime = project.build.runtime or "" if project.build.framework: @@ -129,24 +131,16 @@ def dump_project_yaml(project: MorphProject) -> str: if project.build.package_manager: build_package_manager = project.build.package_manager or "" if project.build.context: - build_context = project.build.context or "# context: ." + build_context = f"{project.build.context}" or "." if project.build.build_args: build_args_items = [] for key, value in project.build.build_args.items(): build_args_items.append(f"{key}={value}") build_args_str = ( - "\n - ".join([""] + build_args_items) + "\n # - ".join([""] + build_args_items) if build_args_items else "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" ) - else: - # Use default BuildConfig - build_use_custom_dockerfile = "false" - build_runtime = "" - build_framework = "" - build_package_manager = "" - build_context = "# context: ." - build_args_str = "\n # - ARG_NAME=value\n # - ANOTHER_ARG=value" # Set values if deployment exists if project.deployment: @@ -176,7 +170,7 @@ def dump_project_yaml(project: MorphProject) -> str: return f""" # Cloud Settings profile: {project.profile} # Defined in the Profile Section in `~/.morph/credentials` -project_id: {project.project_id} +project_id: {project.project_id or ""} # Framework Settings default_connection: {project.default_connection} @@ -192,8 +186,8 @@ def dump_project_yaml(project: MorphProject) -> str: package_manager: {build_package_manager} # pip, poetry, uv # These settings are required when use_custom_dockerfile is true # They define how the Docker image will be built - context: {build_context if build_context != "# context: ." else "# context: ."} - build_args:{build_args_str} + # context: {build_context} + # build_args:{build_args_str} # Deployment Settings deployment: From 86622ec2e6f23a2a399220b97bd7b863b86dffb3 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 16:44:58 +0900 Subject: [PATCH 05/11] change dockerfile api call --- core/morph/config/project.py | 2 +- core/morph/task/deploy.py | 13 +++++++++++-- core/morph/task/utils/load_dockerfile.py | 10 +++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index b873853..36d9e01 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -170,7 +170,7 @@ def dump_project_yaml(project: MorphProject) -> str: return f""" # Cloud Settings profile: {project.profile} # Defined in the Profile Section in `~/.morph/credentials` -project_id: {project.project_id or ""} +project_id: {project.project_id or "null"} # Framework Settings default_connection: {project.default_connection} diff --git a/core/morph/task/deploy.py b/core/morph/task/deploy.py index 8884b3a..8698db6 100644 --- a/core/morph/task/deploy.py +++ b/core/morph/task/deploy.py @@ -56,11 +56,20 @@ def __init__(self, args: Flags): click.echo(click.style(f"Error: {self.dockerfile} not found", fg="red")) sys.exit(1) else: + provider = "aws" + if ( + self.project.deployment is not None + and self.project.deployment.provider is not None + ): + provider = self.project.deployment.provider or "aws" if self.project.build is None: - dockerfile, dockerignore = get_dockerfile_from_api("morph", None, None) + dockerfile, dockerignore = get_dockerfile_from_api( + "morph", provider, None, None + ) else: dockerfile, dockerignore = get_dockerfile_from_api( - self.project.build.framework, + self.project.build.framework or "morph", + provider, self.project.build.package_manager, self.project.build.runtime, ) diff --git a/core/morph/task/utils/load_dockerfile.py b/core/morph/task/utils/load_dockerfile.py index 64c950f..77d8f1f 100644 --- a/core/morph/task/utils/load_dockerfile.py +++ b/core/morph/task/utils/load_dockerfile.py @@ -4,7 +4,8 @@ def get_dockerfile_from_api( - framework: Optional[str] = "morph", + framework: str, + provider: str, package_manager: Optional[str] = None, runtime: Optional[str] = None, ) -> Tuple[str, str]: @@ -13,15 +14,18 @@ def get_dockerfile_from_api( Args: framework: The framework to get the dockerfile for + provider: The provider to get the dockerfile for package_manager: Optional package manager to use - language_version: Optional language version to use + runtime: Optional runtime to use Returns: Tuple containing (dockerfile, dockerignore) """ url = f"https://backend-api-public.morph-cb9.workers.dev/dockerfile/{framework}" - params: Dict[str, Any] = {} + params: Dict[str, Any] = { + "provider": provider, + } if package_manager: params["packageManager"] = package_manager if runtime: From c3a49fe931f14e6a905dba54711a3cab321ee37a Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Fri, 21 Mar 2025 16:49:31 +0900 Subject: [PATCH 06/11] issue v0.3.0rc2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9c1bd08..e5fa843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "morph-data" -version = "0.3.0rc1" +version = "0.3.0rc2" description = "Morph is a python-centric full-stack framework for building and deploying data apps." authors = ["Morph "] packages = [ From dd7f1831bb9d0e5c605e4acc7b2542a761724cc5 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Sat, 22 Mar 2025 14:59:49 +0900 Subject: [PATCH 07/11] change logic for checking use_custom_dockerfile --- core/morph/config/project.py | 11 ++--------- core/morph/task/deploy.py | 13 +++++-------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index 36d9e01..baffd98 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -14,7 +14,6 @@ class BuildConfig(BaseModel): - use_custom_dockerfile: bool = False runtime: Optional[str] = None framework: Optional[str] = "morph" package_manager: Optional[str] = None @@ -101,7 +100,6 @@ def dump_project_yaml(project: MorphProject) -> str: source_paths = "\n- ".join([""] + project.source_paths) # Default values - build_use_custom_dockerfile = "false" build_runtime = "" build_framework = "" build_package_manager = "" @@ -120,10 +118,6 @@ def dump_project_yaml(project: MorphProject) -> str: # Set values if build exists if project.build: - if project.build.use_custom_dockerfile is not None: - build_use_custom_dockerfile = str( - project.build.use_custom_dockerfile - ).lower() if project.build.runtime: build_runtime = project.build.runtime or "" if project.build.framework: @@ -178,13 +172,12 @@ def dump_project_yaml(project: MorphProject) -> str: # Build Settings build: - use_custom_dockerfile: {build_use_custom_dockerfile} - # These settings are required when use_custom_dockerfile is false + # These settings are required when there is no Dockerfile in the project root. # They define the environment in which the project will be built runtime: {build_runtime} # python3.9, python3.10, python3.11, python3.12 framework: {build_framework} package_manager: {build_package_manager} # pip, poetry, uv - # These settings are required when use_custom_dockerfile is true + # These settings are required when there is a Dockerfile in the project root. # They define how the Docker image will be built # context: {build_context} # build_args:{build_args_str} diff --git a/core/morph/task/deploy.py b/core/morph/task/deploy.py index 8698db6..3996ff2 100644 --- a/core/morph/task/deploy.py +++ b/core/morph/task/deploy.py @@ -50,12 +50,9 @@ def __init__(self, args: Flags): self.package_manager = self.project.package_manager # Check Dockerfile existence - self.dockerfile = os.path.join(self.project_root, "Dockerfile") - if self.project.build is not None and self.project.build.use_custom_dockerfile: - if not os.path.exists(self.dockerfile): - click.echo(click.style(f"Error: {self.dockerfile} not found", fg="red")) - sys.exit(1) - else: + self.dockerfile_path = os.path.join(self.project_root, "Dockerfile") + self.use_custom_dockerfile = os.path.exists(self.dockerfile_path) + if self.use_custom_dockerfile: provider = "aws" if ( self.project.deployment is not None @@ -73,7 +70,7 @@ def __init__(self, args: Flags): self.project.build.package_manager, self.project.build.runtime, ) - with open(self.dockerfile, "w") as f: + with open(self.dockerfile_path, "w") as f: f.write(dockerfile) dockerignore_path = os.path.join(self.project_root, ".dockerignore") with open(dockerignore_path, "w") as f: @@ -440,7 +437,7 @@ def _build_docker_image(self) -> str: "-t", self.image_name, "-f", - self.dockerfile, + self.dockerfile_path, self.project_root, ] if self.no_cache: From b508198c9b564691e5197cbaedadeb19daf52aa1 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Sat, 22 Mar 2025 15:14:45 +0900 Subject: [PATCH 08/11] change initial morph_project.yml --- core/morph/config/project.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index baffd98..f600294 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -162,14 +162,16 @@ def dump_project_yaml(project: MorphProject) -> str: deployment_provider = "aws" return f""" -# Cloud Settings -profile: {project.profile} # Defined in the Profile Section in `~/.morph/credentials` -project_id: {project.project_id or "null"} +version: 1 # Framework Settings default_connection: {project.default_connection} source_paths:{source_paths} +# Cloud Settings +# profile: {project.profile} # Defined in the Profile Section in `~/.morph/credentials` +# project_id: {project.project_id or "null"} + # Build Settings build: # These settings are required when there is no Dockerfile in the project root. From 0552e288a436ebeb40ebabf957e8c1100e3a0280 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Sat, 22 Mar 2025 15:20:18 +0900 Subject: [PATCH 09/11] fix version value as string --- core/morph/config/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/morph/config/project.py b/core/morph/config/project.py index f600294..7e6b914 100644 --- a/core/morph/config/project.py +++ b/core/morph/config/project.py @@ -162,7 +162,7 @@ def dump_project_yaml(project: MorphProject) -> str: deployment_provider = "aws" return f""" -version: 1 +version: '1' # Framework Settings default_connection: {project.default_connection} From 521044bb1176020c41db9e8811fe7d3692d8dfe2 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Sun, 23 Mar 2025 14:37:37 +0900 Subject: [PATCH 10/11] update version as v0.3.0rc3 --- core/morph/task/utils/load_dockerfile.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/morph/task/utils/load_dockerfile.py b/core/morph/task/utils/load_dockerfile.py index 77d8f1f..cfbbddb 100644 --- a/core/morph/task/utils/load_dockerfile.py +++ b/core/morph/task/utils/load_dockerfile.py @@ -21,7 +21,7 @@ def get_dockerfile_from_api( Returns: Tuple containing (dockerfile, dockerignore) """ - url = f"https://backend-api-public.morph-cb9.workers.dev/dockerfile/{framework}" + url = f"https://dockerfile-template.morph-cb9.workers.dev/dockerfile/{framework}" params: Dict[str, Any] = { "provider": provider, diff --git a/pyproject.toml b/pyproject.toml index e5fa843..7905855 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "morph-data" -version = "0.3.0rc2" +version = "0.3.0rc3" description = "Morph is a python-centric full-stack framework for building and deploying data apps." authors = ["Morph "] packages = [ From 52e1ceb103e8531a9e505d1044f24454345714d6 Mon Sep 17 00:00:00 2001 From: shibatanaoto Date: Mon, 24 Mar 2025 12:35:13 +0900 Subject: [PATCH 11/11] update version as v0.3.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7905855..a9a39d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "morph-data" -version = "0.3.0rc3" +version = "0.3.0" description = "Morph is a python-centric full-stack framework for building and deploying data apps." authors = ["Morph "] packages = [