fix safe import and base executor test

Author: sokolegg

Makefile

@@ -6,7 +6,7 @@ lint:
 	flake8 piper
 
 unit:
-	#pytest -vs tests/base_test.py
+	pytest -vs tests/base_test.py
 	pytest -vs tests/envs_test.py::TestCompose
 	pytest -vs tests/envs_test.py::TestVenv
 

piper/base/docker/__init__.py

@@ -22,7 +22,10 @@ def render(self):
                                        trim_blocks=True,
                                        lstrip_blocks=True)
         template = jinja_env.get_template(self.template_file)
-        return template.render(cmd=self.cmd, python_docker_version=self.python_docker_version, run_command_lines=self.run_rows, post_install_lines=self.post_install_lines)
+        return template.render(cmd=self.cmd,
+                               python_docker_version=self.python_docker_version,
+                               run_command_lines=self.run_rows,
+                               post_install_lines=self.post_install_lines)
 
 
 # class PythonTesseractImage:

piper/base/executors.py

@@ -4,22 +4,22 @@
 from piper.base.docker import PythonImage
 from piper.configurations import get_configuration
 from piper.envs import get_env, is_current_env, is_docker_env
-from piper.utils import docker_utils as du
+from piper.utils import docker_utils
 
 import inspect
-import os
 import sys
 import time
 from abc import ABC, abstractmethod
-from distutils.command.config import config
 from typing import Dict
+from distutils.dir_util import copy_tree
 
 import aiohttp
 import docker
 import requests
-from loguru import logger
 from pydantic import BaseModel  # , BytesObject, ListOfStringsObject
 
+from piper.utils.logger_utils import logger
+
 
 class BaseExecutor:
     pass
@@ -88,7 +88,6 @@ async def __call__(self, *args, **kwargs):
 
 def copy_piper(path: str):
     cfg = get_configuration()
-    from distutils.dir_util import copy_tree
     copy_tree(cfg.piper_path, f"{path}piper")
 
 
@@ -107,7 +106,8 @@ def write_requirements(path, requirements):
 def build_image(path: str, docker_image):
     client = docker.DockerClient(base_url='unix://var/run/docker.sock')
     image = docker_image.render()
-    with open(f"{path}/Dockerfile", "w") as output:
+    print(f"{path}Dockerfile")
+    with open(f"{path}Dockerfile", "w") as output:
         output.write(image)
 
     image, logs = client.images.build(path=path,
@@ -146,6 +146,7 @@ def wait_for_fast_api_app_start(host, external_port, wait_on_iter, n_iters):
             if r.status_code == 200:
                 break
         except Exception as e:
+            logger.error(f"Exception while starting FastAPI app {e}")
             time.sleep(wait_on_iter)
 
         if i == n_iters:
@@ -170,12 +171,15 @@ def __init__(self, port: int = 8080, **service_kwargs):
 
             copy_piper(project_output_path)
             copy_scripts(project_output_path, self.scripts())
-            # build_image(project_output_path, docker_image)
             self.create_fast_api_files(project_output_path, **service_kwargs)
 
+            docker_image = PythonImage(self.image_tag, "3.9", cmd=f"./run.sh", template_file='default-python.j2',
+                                       run_rows="", post_install_lines="")
+            build_image(project_output_path, docker_image)
+
             # create and run docker container
             # if container exits it will be recreated!
-            du.create_image_and_container_by_dockerfile(
+            docker_utils.create_image_and_container_by_dockerfile(
                 docker_client,
                 project_output_path,
                 self.image_tag,
@@ -206,14 +210,13 @@ def create_fast_api_files(self, path: str, **service_kwargs):
                                           function_name=self.base_handler,
                                           request_model="StringValue",
                                           response_model="StringValue")
-
         with open(f"{path}/main.py", "w") as output:
             output.write(backend)
 
         write_requirements(path, self.requirements)
 
         gunicorn = "#!/bin/bash \n" \
-                   f"gunicorn -b 0.0.0.0:8080 --workers {cfg.n_gunicorn_workers} main:app --worker-class uvicorn.workers.UvicornWorker --preload --timeout 120 --reload=True"
+                   f"gunicorn -b 0.0.0.0:8080 --workers {cfg.n_gunicorn_workers} main:app --worker-class uvicorn.workers.UvicornWorker --preload --timeout 120"
         with open(f"{path}/run.sh", "w") as output:
             output.write(gunicorn)
 
@@ -257,7 +260,7 @@ def __init__(self, port: int = 8080, **service_kwargs):
 
             # create and run docker container
             # if container exits it will be recreated!
-            du.create_image_and_container_by_dockerfile(
+            docker_utils.create_image_and_container_by_dockerfile(
                 docker_client,
                 project_output_path,
                 self.image_tag,

piper/configurations.py

@@ -1,6 +1,9 @@
+import time
+
+
 class Configuration:
-    path: str = "./piper_new_out/"
-    test_path: str = "./piper_test_out/"
+    path: str = f"./applications/piper_project_{time.time_ns()}/"
+    test_path: str = f"./applications/piper_project_{time.time_ns()}/"
     piper_path: str = "piper"
     default_env: str = "docker"
     docker_app_port: int = 8788
@@ -9,7 +12,7 @@ class Configuration:
     number: int = 10
 
     env: str = 'compose'
-    ignore_import_errors: bool = False
+    ignore_import_errors: bool = True
 
     # start time and counter
     wait_on_iter = 0.5

piper/imports.py

@@ -1,7 +1,11 @@
 import builtins
+import inspect
 from types import ModuleType
 
 from piper.utils.logger_utils import logger
+from piper.configurations import get_configuration
+
+configuration = get_configuration()
 
 
 def _empty_import():
@@ -27,13 +31,35 @@ def __getattr__(self, name):
 def try_import(name, globals={}, locals={}, fromlist=[], level=0):
     """
     This import replace real Python import with fake import which returns warning only and PiperDummyModule.
+    This works for everything under piper/ frameworks files by filename but not for piper import (like piper.base)
+    And this also works for every file where you import something from piper firstly !
     """
-    try:
+    is_piper_imported_from_previous_module = False
+    prev_module_locals = inspect.currentframe().f_back.f_locals
+    prev_module_globals = inspect.currentframe().f_back.f_globals
+    prev_modules = prev_module_locals | prev_module_globals
+
+    if prev_modules.values():
+        prev_modules = [v for v in prev_modules.values() if v is not None and hasattr(v, "__module__")]
+        if len(prev_modules) > 0:
+            is_piper_imported_from_previous_module = \
+                any("piper" in v.__module__ for v in prev_modules if type(v.__module__) == str)
+
+    is_import_from_piper_source_code = "__file__" in globals and "piper/" in globals["__file__"]
+    not_piper_import = not ("piper" in name)
+    is_from_source_but_not_piper = is_import_from_piper_source_code and not_piper_import
+
+    if is_piper_imported_from_previous_module and is_from_source_but_not_piper:
+        logger.info(f"Piper activates safe import for library {name} in piper file {globals['__file__']} ")
+        try:
+            return real_import(name, globals, locals, fromlist, level)
+        except ImportError as e:
+            logger.warning(f"Piper ignores ImportError and module {name} "
+                           f": replaced by dummy module. ImportError: {e.with_traceback(None)}")
+            module = PiperDummyModule(name)
+            return module
+    else:
         return real_import(name, globals, locals, fromlist, level)
-    except ImportError as e:
-        logger.warning(f"Piper ignores ImportError and module {name} replaced by dummy module. ImportError: {e}")
-        module = PiperDummyModule(name)
-        return module
 
 
 """

piper/services/__init__.py

@@ -6,14 +6,15 @@
     import spacy
     from fastapi.responses import JSONResponse
 
-from loguru import logger
+# from loguru import logger
+from piper.utils.logger_utils import logger
 from pydantic import BaseModel
 
 from piper.base.executors import FastAPIExecutor, FastAPITesseractExecutor
 from piper.configurations import get_configuration
 from piper.utils import tesrct_utils as tu
 
-logger.add("file.log", level="INFO", backtrace=True, diagnose=True, rotation='5 MB')
+# logger.add("file.log", level="INFO", backtrace=True, diagnose=True, rotation='5 MB')
 
 
 class StringValue(BaseModel):

piper/utils/docker_utils.py

@@ -1,9 +1,11 @@
+from piper.configurations import get_configuration
+
 import sys
 import time
 
 import docker
-from configurations import get_configuration
-from loguru import logger
+# from loguru import logger
+from piper.utils.logger_utils import logger
 
 cfg = get_configuration()
 
@@ -127,7 +129,7 @@ def image_find_and_rm(docker_client, image_tag):
         return True
 
 
-def create_image_and_container_by_dockerfile(docker_client, path, image_tag, container_name, port):
+def create_image_and_container_by_dockerfile(docker_client: docker.DockerClient, path, image_tag, container_name, port):
     # should be deleted
     status = stop_and_rm_container(docker_client, container_name)
 
@@ -156,7 +158,10 @@ def create_image_and_container_by_dockerfile(docker_client, path, image_tag, con
 
             # run container
             try:
-                container = docker_client.containers.run(image_tag, name=container_name, detach=True, ports={8080:port})
+                container = docker_client.containers.run(image_tag,
+                                                         name=container_name,
+                                                         detach=True,
+                                                         ports={8080: port})
                 for log in container.logs():
                     logger.debug(log)
                 logger.info(f'container {container} created')

piper/utils/logger_utils.py

@@ -10,7 +10,7 @@
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 
-fh = logging.FileHandler('../piper_base_log.log')
+fh = logging.FileHandler('./piper_base_log.log')
 fh.setLevel(logging.DEBUG)
 fh.setFormatter(formatter)
 logger.addHandler(fh)

piper/utils/tesrct_utils.py

@@ -6,7 +6,8 @@
     import pytesseract
 
 import requests
-from loguru import logger
+# from loguru import logger
+from piper.utils.logger_utils import logger
 
 from piper.configurations import get_configuration
 

tests/base_test.py

@@ -1,29 +1,25 @@
 # pytest -vs tests/base_test.py::TestPiperBase
 import asyncio
-import os
-import sys
-import time
 
-root_dir = os.path.join(os.path.realpath(os.path.pardir), 'piper')
-sys.path.insert(1, root_dir)
+from piper.base.executors import FastAPIExecutor
+from piper.services import StringValue
+from piper.configurations import get_configuration
+from piper.envs import CurrentEnv, DockerEnv, ComposeEnv
 
 
-class TestPiperBase:
+class TestMessageAdder(FastAPIExecutor):
 
-    def test_simple_executor(self):
-        from piper.base.executors import FastAPIExecutor
-        from piper.services import StringValue
-        from piper.configurations import get_configuration
-        from piper.envs import CurrentEnv, DockerEnv
+    def __init__(self, appender="TEST", **kwargs):
+        self.appender = appender
+        super().__init__(**kwargs)
 
-        class TestMessageAdder(FastAPIExecutor):
+    async def run(self, message: StringValue) -> StringValue:
+        return StringValue(value=(message.value + self.appender))
 
-            def __init__(self, appender="TEST", **kwargs):
-                self.appender = appender
-                super().__init__(**kwargs)
 
-            async def run(self, message: StringValue) -> StringValue:
-                return StringValue(value=(message.value + self.appender))
+class TestPiperBase:
+
+    def test_simple_executor(self):
 
         cfg = get_configuration()
         loop = asyncio.get_event_loop()
@@ -32,15 +28,15 @@ async def run(self, message: StringValue) -> StringValue:
         need_result = f'{init_value}TEST'
 
         with CurrentEnv() as env:
-            adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
+            adder = TestMessageAdder(port=cfg.docker_app_port)
             result = loop.run_until_complete(adder(x))
             print(result)
-            assert result.get('value') == need_result
+            assert result.value == need_result
 
         with DockerEnv() as env:
-            adder = TestMessageAdder(appender="!", port=cfg.docker_app_port)
+            adder = TestMessageAdder(port=cfg.docker_app_port)
             result = loop.run_until_complete(adder(x))
             print(result)
             adder.rm_container()
-            assert result.get('value') == need_result
+            assert result.get("value") == need_result