diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index a47c5b7f..cafb30b0 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -8,7 +8,7 @@ echo 'export PATH="/root/.local/bin:$PATH"' > ~/.bashrc sudo `which poetry` config virtualenvs.create false sudo `which poetry` install --with dev python scripts/fetch_core.py -python scripts/zip_templates.py +python scripts/bundle_resources.py playwright install-deps playwright install # Run mypy once so that it will install any needed type stubs. After this, the VSCode extension will run it automatically. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 04309693..976adabd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install -e . python scripts/fetch_core.py - python scripts/zip_templates.py + python scripts/bundle_resources.py - name: Check formatting with black run: python -m black --check --diff $(git ls-files "*.py") - name: Check for lint @@ -75,7 +75,7 @@ jobs: run: | python -m poetry install python -m poetry run python scripts/fetch_core.py - python -m poetry run python scripts/zip_templates.py + python -m poetry run python scripts/bundle_resources.py - name: Test with pytest run: | diff --git a/.gitignore b/.gitignore index e8d683ee..d19fbc27 100644 --- a/.gitignore +++ b/.gitignore @@ -138,11 +138,8 @@ cython_debug/ #pretext-core pretext/core/pretext.py -pretext/core/resources.zip -#shipped templates -pretext/templates/resources -#old "static" stuff (deprecated -pretext/static +#zipped resources +pretext/resources/*.zip #default new pretext project new-pretext-project diff --git a/pretext/__init__.py b/pretext/__init__.py index 1254a9e7..f35220b7 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -21,7 +21,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "ed4e9d94a75b5b728aca2eb06dfde3f1595ab783" +CORE_COMMIT = "ac1ca3ca67c9512059afd6fd37a714a7fc988a5d" def activate() -> None: diff --git a/pretext/cli.py b/pretext/cli.py index 6645377d..fb7a84d6 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -25,7 +25,7 @@ from . import ( utils, - templates, + resources, core, constants, plastex, @@ -248,7 +248,8 @@ def devscript(args: List[str]) -> None: """ PY_CMD = sys.executable subprocess.run( - [PY_CMD, str(core.resources.path("pretext", "pretext"))] + list(args) + [PY_CMD, str(resources.resource_base_path() / "core" / "pretext" / "pretext")] + + list(args) ) @@ -284,47 +285,35 @@ def new(template: str, directory: Path, url_template: str) -> None: """ directory_fullpath = Path(directory).resolve() if utils.project_path(directory_fullpath) is not None: - log.warning( + log.error( f"A project already exists in `{utils.project_path(directory_fullpath)}`." ) - log.warning("No new project will be generated.") + log.error("No new project will be generated.") return - log.info( - f"Generating new PreTeXt project in `{directory_fullpath}` using `{template}` template." - ) + log.info(f"Generating new PreTeXt project in `{directory_fullpath}`") + directory_fullpath.mkdir(exist_ok=True) if url_template is not None: + log.info(f"Using template at `{url_template}`") + # get project and extract to directory r = requests.get(url_template) archive = zipfile.ZipFile(io.BytesIO(r.content)) + with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: + archive.extractall(tmpdirname) + content_path = [Path(tmpdirname) / i for i in os.listdir(tmpdirname)][0] + shutil.copytree(content_path, directory_fullpath, dirs_exist_ok=True) else: - with templates.resource_path(f"{template}.zip") as template_path: - archive = zipfile.ZipFile(template_path) - # find (first) project.ptx to use as root of template - filenames = [Path(filepath).name for filepath in archive.namelist()] - project_ptx_index = filenames.index("project.ptx") - project_ptx_path = Path(archive.namelist()[project_ptx_index]) - project_dir_path = project_ptx_path.parent - with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: - temp_path = Path(tmpdirname) / "new-project" - temp_path.mkdir() - for filepath in [ - filepath - for filepath in archive.namelist() - if project_dir_path in Path(filepath).parents - ]: - archive.extract(filepath, path=temp_path) - tmpsubdirname = temp_path / project_dir_path - shutil.copytree(tmpsubdirname, directory_fullpath, dirs_exist_ok=True) - # generate remaining boilerplate like requirements.txt - project = Project.parse(directory_fullpath) - project.generate_boilerplate(update_requirements=True) - if len(project.targets) == 0: - log.warning("The generated project has no targets!") - else: - target = project.targets[0] - log.info(f"Success! Open `{target.source_abspath()}` to edit your document") - log.info( - f"Then try to `pretext build` and `pretext view` from within `{directory_fullpath}`." - ) + log.info(f"Using `{template}` template.") + # copy project from installed resources + with resources.resource_base_path() / "templates" / f"{template}" as template_path: + shutil.copytree(template_path, directory_fullpath, dirs_exist_ok=True) + # generate missing boilerplate + with utils.working_directory(directory_fullpath): + project_path = utils.project_path() + if project_path is None: + project = Project() + else: + project = Project.parse(project_path) + project.generate_boilerplate(update_requirements=True) # pretext init diff --git a/pretext/core/__init__.py b/pretext/core/__init__.py index d340a75b..c721554c 100644 --- a/pretext/core/__init__.py +++ b/pretext/core/__init__.py @@ -7,10 +7,10 @@ "Run `scripts/fetch_core.py` to grab a copy of pretex core.\n" "The original error message is: " + e.msg ) -from . import resources +from .. import resources from .. import CORE_COMMIT, VERSION -set_ptx_path(resources.path()) +set_ptx_path(resources.resource_base_path() / "core") def cli_build_message() -> str: diff --git a/pretext/core/resources.py b/pretext/core/resources.py deleted file mode 100644 index 2aed8f9f..00000000 --- a/pretext/core/resources.py +++ /dev/null @@ -1,43 +0,0 @@ -from pathlib import Path -import zipfile -import importlib.resources -import shutil -from .. import CORE_COMMIT - - -def path(*args: str) -> Path: - # Checks that the local static path ~/.ptx/ contains the static files needed for core, and installs them if they are missing (or if the version is different from the installed version of pretext). Then returns the absolute path to the static files (appending arguments) - local_base_path = Path.home() / ".ptx" - local_commit_file = Path(local_base_path) / ".commit" - if not Path.is_file(local_commit_file): - print("Static pretext files do not appear to be installed. Installing now.") - install(local_base_path) - # check that the static core_commit matches current core_commit - with open(local_commit_file, "r") as f: - static_commit = f.readline().strip() - if static_commit != CORE_COMMIT: - print("Static pretext files are out of date. Installing them now.") - install(local_base_path) - return local_base_path.joinpath(*args) - - -def install(local_base_path: Path) -> None: - backup_dir = local_base_path.with_name(local_base_path.name + ".bak") - if Path.is_dir(backup_dir): - # remove old backup: - shutil.rmtree(backup_dir) - if Path.is_dir(local_base_path): - print(f"Backing up old static files to {backup_dir}") - Path.rename(local_base_path, backup_dir) - Path.mkdir(local_base_path) - - with importlib.resources.path("pretext.core", "resources.zip") as static_zip: - with zipfile.ZipFile(static_zip, "r") as zip: - zip.extractall(local_base_path) - # Write the current commit to local file - with open(local_base_path / ".commit", "w") as f: - f.write(CORE_COMMIT) - print( - f"Static files required for pretext have now been installed to {local_base_path}" - ) - return diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 42422130..db4167f3 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -24,7 +24,7 @@ from .. import codechat from .. import utils from .. import types as pt # PreTeXt types -from .. import templates +from ..resources import resource_base_path from .. import VERSION @@ -286,7 +286,7 @@ def post_validate(self) -> None: if not self.publication_abspath().exists(): # ... then use the CLI's built-in template file. # TODO: this is wrong, since the returned path is only valid inside the context manager. Instead, need to enter the context here, then exit it when this class is deleted (also problematic). - with templates.resource_path("publication.ptx") as self.publication: + with resource_base_path() / "templates" / "publication.ptx" as self.publication: pass # Otherwise, verify that the provided publication file exists. TODO: It is silly to check that all publication files exist. We warn when they don't. If the target we are calling has a non-existent publication file, then that error will be caught anyway. else: @@ -1455,7 +1455,7 @@ def generate_boilerplate( f"Your existing {resource} file has been backed up at {backup_resource_path}." ) if resource != "requirements.txt": - with templates.resource_path(resource) as resource_path: + with resource_base_path() / "templates" / resource as resource_path: if ( not project_resource_path.exists() or resource_path.read_text() diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py new file mode 100644 index 00000000..0c271990 --- /dev/null +++ b/pretext/resources/__init__.py @@ -0,0 +1,43 @@ +import importlib.resources +import logging +from pathlib import Path +import shutil +import zipfile + +from .. import VERSION, CORE_COMMIT + +log = logging.getLogger("ptxlogger") + +_RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION + + +def install(reinstall: bool = False) -> None: + if _RESOURCE_BASE_PATH.exists(): + if reinstall: + log.info(f"Deleting existing resources at {_RESOURCE_BASE_PATH}") + shutil.rmtree(_RESOURCE_BASE_PATH) + else: + log.warning(f"Resources are already installed at {_RESOURCE_BASE_PATH}") + return + _RESOURCE_BASE_PATH.mkdir(parents=True) + + log.info("Installing core resources") + with importlib.resources.path("pretext.resources", "core.zip") as static_zip: + with zipfile.ZipFile(static_zip, "r") as zip: + zip.extractall(path=_RESOURCE_BASE_PATH) + (_RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( + _RESOURCE_BASE_PATH / "core" + ) + + log.info("Installing templates") + (_RESOURCE_BASE_PATH / "templates").mkdir() + with importlib.resources.path("pretext.resources", "templates.zip") as static_zip: + with zipfile.ZipFile(static_zip, "r") as zip: + zip.extractall(path=_RESOURCE_BASE_PATH / "templates") + + +def resource_base_path() -> Path: + if not _RESOURCE_BASE_PATH.exists(): + log.info(f"Installing resources to {_RESOURCE_BASE_PATH}") + install() + return _RESOURCE_BASE_PATH diff --git a/pretext/templates/__init__.py b/pretext/templates/__init__.py deleted file mode 100644 index aa690e48..00000000 --- a/pretext/templates/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations -from contextlib import AbstractContextManager -from pathlib import Path -import importlib.resources as ir - - -def resource_path(filename: str) -> AbstractContextManager[Path]: - """ - Returns resource manager - Usage: - with resource_path('foo.bar') as filepath: - # do things - """ - from . import resources - - # Raise FileNotFoundError if path DNE (which happens automatically in some environments anyway, so let's make sure it's consistent for testing.) - with ir.path(resources, filename) as path: - if not path.exists(): - raise FileNotFoundError( - f"Resource `{filename}` does not exist; no such file or directory: {path}" - ) - # Try except here is to use newer importlib function, - # supported starting with 3.9, and required with 3.11 - try: - # TODO: remove the ignore when Python 3.8 support ends. This fails Python 3.8 type checks, since these functions depend on newer Python versions. - return ir.as_file(ir.files(resources).joinpath(filename)) # type: ignore[attr-defined] - except AttributeError: - return ir.path(resources, filename) diff --git a/pretext/utils.py b/pretext/utils.py index cb93278c..0c3ebb5e 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -19,7 +19,7 @@ from typing import Any, cast, List, Optional -from . import core, templates, constants +from . import core, constants, resources # Get access to logger log = logging.getLogger("ptxlogger") @@ -94,7 +94,7 @@ def project_xml(dirpath: t.Optional[Path] = None) -> _ElementTree: dirpath = Path() # current directory pp = project_path(dirpath) if pp is None: - with templates.resource_path("project.ptx") as project_manifest: + with resources.resource_base_path() / "templates" / "project.ptx" as project_manifest: return ET.parse(project_manifest) else: project_manifest = pp / "project.ptx" @@ -173,7 +173,7 @@ def xml_syntax_is_valid(xmlfile: Path, root_tag: str = "pretext") -> bool: def xml_source_validates_against_schema(xmlfile: Path) -> bool: # get path to RelaxNG schema file: - schemarngfile = core.resources.path("schema", "pretext.rng") + schemarngfile = resources.resource_base_path() / "core" / "schema" / "pretext.rng" # Open schemafile for validation: relaxng = ET.RelaxNG(file=schemarngfile) @@ -295,7 +295,9 @@ def copy_custom_xsl(xsl_path: Path, output_dir: Path) -> None: log.debug(f"Copying all files in {xsl_dir} to {output_dir}") shutil.copytree(xsl_dir, output_dir, dirs_exist_ok=True) log.debug(f"Copying core XSL to {output_dir}/core") - shutil.copytree(core.resources.path("xsl"), output_dir / "core") + shutil.copytree( + resources.resource_base_path() / "core" / "xsl", output_dir / "core" + ) def check_executable(exec_name: str) -> Optional[str]: @@ -439,7 +441,9 @@ def show_target_hints( def npm_install() -> None: - with working_directory(core.resources.path("script", "mjsre")): + with working_directory( + resources.resource_base_path() / "core" / "script" / "mjsre" + ): log.info("Attempting to install/update required node packages.") try: subprocess.run("npm install", shell=True) diff --git a/scripts/build_package.py b/scripts/build_package.py index 3ffe474b..6c931212 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -1,6 +1,6 @@ import subprocess import fetch_core -import zip_templates +import bundle_resources def main() -> None: @@ -10,7 +10,7 @@ def main() -> None: # ensure up-to-date "static" resources fetch_core.main() - zip_templates.main() + bundle_resources.main() # Build package subprocess.run(["poetry", "build"], shell=True) diff --git a/scripts/bundle_resources.py b/scripts/bundle_resources.py new file mode 100644 index 00000000..4d467c4c --- /dev/null +++ b/scripts/bundle_resources.py @@ -0,0 +1,16 @@ +import shutil +from pathlib import Path + + +def main() -> None: + shutil.make_archive( + str(Path("pretext") / "resources" / "templates"), "zip", Path("templates") + ) + print("Templates successfully zipped.") + # TODO: incorporate in pelican branch + # shutil.make_archive(str(Path("pretext") / "resources" / "pelican"), 'zip', Path("pelican")) + # print("Pelican resources successfully zipped.") + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch_core.py b/scripts/fetch_core.py index f94471fb..778bb3b5 100644 --- a/scripts/fetch_core.py +++ b/scripts/fetch_core.py @@ -1,44 +1,32 @@ +from pathlib import Path import requests -import zipfile -import io import shutil import tempfile +import zipfile + from pretext import CORE_COMMIT -from pathlib import Path -from remove_path import remove_path def main() -> None: # grab copy of necessary PreTeXtBook/pretext files from specified commit print(f"Requesting core PreTeXtBook/pretext commit {CORE_COMMIT} from GitHub.") - pretext_dir = Path("pretext").resolve() + core_zip_path = Path("pretext").resolve() / "resources" / "core.zip" r = requests.get( f"https://github.com/PreTeXtBook/pretext/archive/{CORE_COMMIT}.zip" ) - archive = zipfile.ZipFile(io.BytesIO(r.content)) + with open(core_zip_path, "wb") as f: + f.write(r.content) with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: - archive.extractall(tmpdirname) - print("Creating zip of static folders") - # Copy required folders to a single folder to be zipped: - for subdir in ["xsl", "schema", "script", "css", "js", "js_lib", "pretext"]: - shutil.copytree( - Path(tmpdirname) / f"pretext-{CORE_COMMIT}" / subdir, - Path(tmpdirname) / "static" / subdir, + with zipfile.ZipFile(core_zip_path) as archive: + pretext_py = archive.extract( + f"pretext-{CORE_COMMIT}/pretext/pretext.py", + path=tmpdirname, ) - shutil.make_archive( - "pretext/core/resources", "zip", Path(tmpdirname) / "static" - ) - print("Copying new version of pretext.py to core directory") - remove_path(pretext_dir / "core" / "pretext.py") + assert Path(pretext_py).exists() shutil.copyfile( - Path(tmpdirname).resolve() - / f"pretext-{CORE_COMMIT}" - / "pretext" - / "pretext.py", - Path("pretext") / "core" / "pretext.py", + Path(pretext_py), Path("pretext").resolve() / "core" / "pretext.py" ) - print("Successfully updated core PreTeXtBook/pretext resources from GitHub.") diff --git a/scripts/symlink_core.py b/scripts/symlink_core.py index 6550299b..7617108a 100644 --- a/scripts/symlink_core.py +++ b/scripts/symlink_core.py @@ -1,21 +1,17 @@ +import shutil import sys from pathlib import Path -from remove_path import remove_path -import pretext.core.resources +import pretext.resources # This will redirect the static resources for pretext, including the core python script, xsl, css, etc to a local directory of your choosing that contains the clone of the pretext repository. This is useful for development purposes, as it allows you to make changes to the core python script and test with the CLI as you normally would. def main(core_path: Path = Path("../pretext")) -> None: - for subdir in ["xsl", "schema", "script", "css", "js", "js_lib"]: - original_path = (core_path / subdir).resolve() - link_path = pretext.core.resources.path(subdir) - remove_path(link_path) - link_path.symlink_to(original_path) - original_path = (core_path / "pretext" / "pretext.py").resolve() - link_path = Path("pretext") / "core" / "pretext.py" - remove_path(link_path) - link_path.symlink_to(original_path) - + link_path = pretext.resources.resource_base_path() / "core" + if link_path.is_dir(): + shutil.rmtree(link_path) + else: + link_path.unlink() + link_path.symlink_to(core_path) print(f"Linked local core pretext directory `{core_path}`") diff --git a/scripts/zip_templates.py b/scripts/zip_templates.py deleted file mode 100644 index 175902af..00000000 --- a/scripts/zip_templates.py +++ /dev/null @@ -1,57 +0,0 @@ -import shutil -import glob -import tempfile -from pathlib import Path - - -def main() -> None: - static_template_path = Path("pretext") / "templates" / "resources" - print(f"Zipping templates from source into `{static_template_path}`.") - - for template_directory in glob.iglob("templates/[!.]*"): - template_path = Path(template_directory) - if template_path.is_dir(): - with tempfile.TemporaryDirectory(prefix="pretext_") as temporary_directory: - temporary_path = Path(temporary_directory) - shutil.copytree( - template_path, - temporary_path, - dirs_exist_ok=True, - ) - template_files = [ - "project.ptx", - ".gitignore", - "codechat_config.yaml", - ".devcontainer.json", - ] - for template_file in template_files: - copied_template_file = temporary_path / template_file - if not copied_template_file.is_file(): - shutil.copyfile( - Path("templates") / template_file, - copied_template_file, - ) - template_zip_basename = template_path.name - shutil.make_archive( - str(static_template_path / template_zip_basename), - "zip", - temporary_path, - ) - for f in [ - "codechat_config.yaml", - "project.ptx", - "publication.ptx", - ".gitignore", - ".devcontainer.json", - "pretext-cli.yml", - ]: - shutil.copyfile(Path("templates") / f, static_template_path / f) - - with open(static_template_path / "__init__.py", "w") as _: - pass - - print(f"Templates successfully zipped into `{static_template_path}`.") - - -if __name__ == "__main__": - main() diff --git a/tests/test_project.py b/tests/test_project.py index 40b16003..9a9e59f1 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -8,8 +8,8 @@ import pytest from pretext import project as pr -from pretext import templates from pretext import utils +from pretext.resources import resource_base_path from .common import DEMO_MAPPING, check_installed @@ -36,8 +36,6 @@ def test_defaults(tmp_path: Path) -> None: project = pr.Project(ptx_version="2") for t in ts: ts_dict[t[0]] = project.new_target(*t) - with templates.resource_path("publication.ptx") as pub_path: - pass assert project._path == Path.cwd() / Path("project.ptx") assert project.source == Path("source") assert project.publication == Path("publication") @@ -51,7 +49,10 @@ def test_defaults(tmp_path: Path) -> None: assert target.name == name assert target.format == format assert target.source == Path("main.ptx") - assert target.publication == pub_path + assert ( + target.publication + == resource_base_path() / "templates" / "publication.ptx" + ) assert target.output_dir == Path(name) assert target.deploy_dir is None assert target.xsl is None diff --git a/tests/test_sample_article.py b/tests/test_sample_article.py new file mode 100644 index 00000000..080c58ff --- /dev/null +++ b/tests/test_sample_article.py @@ -0,0 +1,25 @@ +import shutil +import pytest +from pathlib import Path +import errorhandler # type: ignore +from pretext.project import Project +from pretext.resources import resource_base_path +import pretext.utils +from .common import check_installed + + +@pytest.mark.skipif( + not check_installed(["xelatex", "--version"]), + reason="Note: several tests are skipped, since xelatex wasn't installed.", +) +def test_sample_article(tmp_path: Path) -> None: + error_checker = errorhandler.ErrorHandler(logger="ptxlogger") + prj_path = tmp_path / "sample" + shutil.copytree( + resource_base_path() / "core" / "examples" / "sample-article", prj_path + ) + with pretext.utils.working_directory(prj_path): + project = Project.parse() + t = project.get_target() + t.build() + assert not error_checker.fired diff --git a/tests/test_templates.py b/tests/test_templates.py deleted file mode 100644 index efafb900..00000000 --- a/tests/test_templates.py +++ /dev/null @@ -1,24 +0,0 @@ -from pretext.templates import resource_path - - -def test_resource_path() -> None: - resources = [ - ".devcontainer.json", - ".gitignore", - "article.zip", - "book.zip", - "codechat_config.yaml", - "demo.zip", - "hello.zip", - "project.ptx", - "publication.ptx", - "slideshow.zip", - ] - for filename in resources: - with resource_path(filename) as path: - assert path.name == filename - try: - with resource_path("does-not-exist.foo-bar") as path: - assert False - except FileNotFoundError: - assert True