From bdc613f35d18344afe722937578f2b8edcd61154 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 17:34:10 +0000 Subject: [PATCH 01/21] rename zip_templates to bundle_resources --- .devcontainer/postCreateCommand.sh | 2 +- .github/workflows/tests.yml | 4 ++-- scripts/build_package.py | 4 ++-- scripts/{zip_templates.py => bundle_resources.py} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename scripts/{zip_templates.py => bundle_resources.py} (100%) diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index a47c5b7f..cafb30b0 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -8,7 +8,7 @@ echo 'export PATH="/root/.local/bin:$PATH"' > ~/.bashrc sudo `which poetry` config virtualenvs.create false sudo `which poetry` install --with dev python scripts/fetch_core.py -python scripts/zip_templates.py +python scripts/bundle_resources.py playwright install-deps playwright install # Run mypy once so that it will install any needed type stubs. After this, the VSCode extension will run it automatically. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 04309693..976adabd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: run: | python -m pip install -e . python scripts/fetch_core.py - python scripts/zip_templates.py + python scripts/bundle_resources.py - name: Check formatting with black run: python -m black --check --diff $(git ls-files "*.py") - name: Check for lint @@ -75,7 +75,7 @@ jobs: run: | python -m poetry install python -m poetry run python scripts/fetch_core.py - python -m poetry run python scripts/zip_templates.py + python -m poetry run python scripts/bundle_resources.py - name: Test with pytest run: | diff --git a/scripts/build_package.py b/scripts/build_package.py index 3ffe474b..2b6710ac 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -1,6 +1,6 @@ import subprocess import fetch_core -import zip_templates +import scripts.bundle_resources as bundle_resources def main() -> None: @@ -10,7 +10,7 @@ def main() -> None: # ensure up-to-date "static" resources fetch_core.main() - zip_templates.main() + bundle_resources.main() # Build package subprocess.run(["poetry", "build"], shell=True) diff --git a/scripts/zip_templates.py b/scripts/bundle_resources.py similarity index 100% rename from scripts/zip_templates.py rename to scripts/bundle_resources.py From bb5d7dded724565532a342f03d3c012b4227d136 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 19:55:32 +0000 Subject: [PATCH 02/21] wip --- .gitignore | 7 ++--- pretext/core/__init__.py | 4 +-- pretext/core/resources.py | 43 ---------------------------- pretext/resources/__init__.py | 39 ++++++++++++++++++++++++++ pretext/templates/__init__.py | 28 ------------------ scripts/bundle_resources.py | 53 ++++------------------------------- scripts/fetch_core.py | 38 ++++++++----------------- 7 files changed, 60 insertions(+), 152 deletions(-) delete mode 100644 pretext/core/resources.py create mode 100644 pretext/resources/__init__.py delete mode 100644 pretext/templates/__init__.py diff --git a/.gitignore b/.gitignore index e8d683ee..d19fbc27 100644 --- a/.gitignore +++ b/.gitignore @@ -138,11 +138,8 @@ cython_debug/ #pretext-core pretext/core/pretext.py -pretext/core/resources.zip -#shipped templates -pretext/templates/resources -#old "static" stuff (deprecated -pretext/static +#zipped resources +pretext/resources/*.zip #default new pretext project new-pretext-project diff --git a/pretext/core/__init__.py b/pretext/core/__init__.py index d340a75b..9b876730 100644 --- a/pretext/core/__init__.py +++ b/pretext/core/__init__.py @@ -7,10 +7,10 @@ "Run `scripts/fetch_core.py` to grab a copy of pretex core.\n" "The original error message is: " + e.msg ) -from . import resources +from .. import resources from .. import CORE_COMMIT, VERSION -set_ptx_path(resources.path()) +set_ptx_path(resources.path("core")) def cli_build_message() -> str: diff --git a/pretext/core/resources.py b/pretext/core/resources.py deleted file mode 100644 index 2aed8f9f..00000000 --- a/pretext/core/resources.py +++ /dev/null @@ -1,43 +0,0 @@ -from pathlib import Path -import zipfile -import importlib.resources -import shutil -from .. import CORE_COMMIT - - -def path(*args: str) -> Path: - # Checks that the local static path ~/.ptx/ contains the static files needed for core, and installs them if they are missing (or if the version is different from the installed version of pretext). Then returns the absolute path to the static files (appending arguments) - local_base_path = Path.home() / ".ptx" - local_commit_file = Path(local_base_path) / ".commit" - if not Path.is_file(local_commit_file): - print("Static pretext files do not appear to be installed. Installing now.") - install(local_base_path) - # check that the static core_commit matches current core_commit - with open(local_commit_file, "r") as f: - static_commit = f.readline().strip() - if static_commit != CORE_COMMIT: - print("Static pretext files are out of date. Installing them now.") - install(local_base_path) - return local_base_path.joinpath(*args) - - -def install(local_base_path: Path) -> None: - backup_dir = local_base_path.with_name(local_base_path.name + ".bak") - if Path.is_dir(backup_dir): - # remove old backup: - shutil.rmtree(backup_dir) - if Path.is_dir(local_base_path): - print(f"Backing up old static files to {backup_dir}") - Path.rename(local_base_path, backup_dir) - Path.mkdir(local_base_path) - - with importlib.resources.path("pretext.core", "resources.zip") as static_zip: - with zipfile.ZipFile(static_zip, "r") as zip: - zip.extractall(local_base_path) - # Write the current commit to local file - with open(local_base_path / ".commit", "w") as f: - f.write(CORE_COMMIT) - print( - f"Static files required for pretext have now been installed to {local_base_path}" - ) - return diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py new file mode 100644 index 00000000..9a67521b --- /dev/null +++ b/pretext/resources/__init__.py @@ -0,0 +1,39 @@ +import importlib.resources +import logging +from pathlib import Path +import shutil +import typing as t +import zipfile + +from .. import VERSION, CORE_COMMIT + +log = logging.getLogger("ptxlogger") + +RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION + + +def path(resource_type: t.Literal["core", "templates"]) -> Path: + return Path(RESOURCE_BASE_PATH / resource_type) + + +def install(reinstall=False) -> None: + if RESOURCE_BASE_PATH.exists(): + if reinstall: + log.info(f"Deleting existing resources at {RESOURCE_BASE_PATH}") + shutil.rmtree(RESOURCE_BASE_PATH) + else: + log.warning(f"Resources are already installed at {RESOURCE_BASE_PATH}") + return + RESOURCE_BASE_PATH.mkdir(parents=True) + + log.info("Installing core resources") + with importlib.resources.path("pretext.resources", "core.zip") as static_zip: + with zipfile.ZipFile(static_zip, "r") as zip: + zip.extractall(path=RESOURCE_BASE_PATH) + (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(RESOURCE_BASE_PATH / "core") + + log.info("Installing templates") + with importlib.resources.path("pretext.resources", "templates.zip") as static_zip: + with zipfile.ZipFile(static_zip, "r") as zip: + zip.extractall(path=RESOURCE_BASE_PATH) + (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(RESOURCE_BASE_PATH / "core") diff --git a/pretext/templates/__init__.py b/pretext/templates/__init__.py deleted file mode 100644 index aa690e48..00000000 --- a/pretext/templates/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations -from contextlib import AbstractContextManager -from pathlib import Path -import importlib.resources as ir - - -def resource_path(filename: str) -> AbstractContextManager[Path]: - """ - Returns resource manager - Usage: - with resource_path('foo.bar') as filepath: - # do things - """ - from . import resources - - # Raise FileNotFoundError if path DNE (which happens automatically in some environments anyway, so let's make sure it's consistent for testing.) - with ir.path(resources, filename) as path: - if not path.exists(): - raise FileNotFoundError( - f"Resource `{filename}` does not exist; no such file or directory: {path}" - ) - # Try except here is to use newer importlib function, - # supported starting with 3.9, and required with 3.11 - try: - # TODO: remove the ignore when Python 3.8 support ends. This fails Python 3.8 type checks, since these functions depend on newer Python versions. - return ir.as_file(ir.files(resources).joinpath(filename)) # type: ignore[attr-defined] - except AttributeError: - return ir.path(resources, filename) diff --git a/scripts/bundle_resources.py b/scripts/bundle_resources.py index 175902af..e705d124 100644 --- a/scripts/bundle_resources.py +++ b/scripts/bundle_resources.py @@ -1,56 +1,13 @@ import shutil -import glob -import tempfile from pathlib import Path def main() -> None: - static_template_path = Path("pretext") / "templates" / "resources" - print(f"Zipping templates from source into `{static_template_path}`.") - - for template_directory in glob.iglob("templates/[!.]*"): - template_path = Path(template_directory) - if template_path.is_dir(): - with tempfile.TemporaryDirectory(prefix="pretext_") as temporary_directory: - temporary_path = Path(temporary_directory) - shutil.copytree( - template_path, - temporary_path, - dirs_exist_ok=True, - ) - template_files = [ - "project.ptx", - ".gitignore", - "codechat_config.yaml", - ".devcontainer.json", - ] - for template_file in template_files: - copied_template_file = temporary_path / template_file - if not copied_template_file.is_file(): - shutil.copyfile( - Path("templates") / template_file, - copied_template_file, - ) - template_zip_basename = template_path.name - shutil.make_archive( - str(static_template_path / template_zip_basename), - "zip", - temporary_path, - ) - for f in [ - "codechat_config.yaml", - "project.ptx", - "publication.ptx", - ".gitignore", - ".devcontainer.json", - "pretext-cli.yml", - ]: - shutil.copyfile(Path("templates") / f, static_template_path / f) - - with open(static_template_path / "__init__.py", "w") as _: - pass - - print(f"Templates successfully zipped into `{static_template_path}`.") + shutil.make_archive(str(Path("pretext") / "resources" / "templates"), 'zip', Path("templates")) + print("Templates successfully zipped.") + # TODO: incorporate in pelican branch + # shutil.make_archive(str(Path("pretext") / "resources" / "pelican"), 'zip', Path("pelican")) + # print("Pelican resources successfully zipped.") if __name__ == "__main__": diff --git a/scripts/fetch_core.py b/scripts/fetch_core.py index f94471fb..50d1a030 100644 --- a/scripts/fetch_core.py +++ b/scripts/fetch_core.py @@ -1,44 +1,30 @@ +from pathlib import Path import requests -import zipfile -import io import shutil import tempfile +import zipfile + from pretext import CORE_COMMIT -from pathlib import Path -from remove_path import remove_path def main() -> None: # grab copy of necessary PreTeXtBook/pretext files from specified commit print(f"Requesting core PreTeXtBook/pretext commit {CORE_COMMIT} from GitHub.") - pretext_dir = Path("pretext").resolve() + core_zip_path = Path("pretext").resolve() / "resources" / "core.zip" r = requests.get( f"https://github.com/PreTeXtBook/pretext/archive/{CORE_COMMIT}.zip" ) - archive = zipfile.ZipFile(io.BytesIO(r.content)) + with open(core_zip_path, 'wb') as f: + f.write(r.content) with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: - archive.extractall(tmpdirname) - print("Creating zip of static folders") - # Copy required folders to a single folder to be zipped: - for subdir in ["xsl", "schema", "script", "css", "js", "js_lib", "pretext"]: - shutil.copytree( - Path(tmpdirname) / f"pretext-{CORE_COMMIT}" / subdir, - Path(tmpdirname) / "static" / subdir, + with zipfile.ZipFile(core_zip_path) as archive: + pretext_py = archive.extract( + f"pretext-{CORE_COMMIT}/pretext/pretext.py", + path=tmpdirname, ) - shutil.make_archive( - "pretext/core/resources", "zip", Path(tmpdirname) / "static" - ) - print("Copying new version of pretext.py to core directory") - remove_path(pretext_dir / "core" / "pretext.py") - shutil.copyfile( - Path(tmpdirname).resolve() - / f"pretext-{CORE_COMMIT}" - / "pretext" - / "pretext.py", - Path("pretext") / "core" / "pretext.py", - ) - + assert Path(pretext_py).exists() + shutil.copyfile(Path(pretext_py), Path("pretext").resolve() / "core" / "pretext.py") print("Successfully updated core PreTeXtBook/pretext resources from GitHub.") From 032d815a960338feee1883832b7e55abb6b02177 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 19:56:56 +0000 Subject: [PATCH 03/21] format --- pretext/resources/__init__.py | 8 ++++++-- scripts/bundle_resources.py | 4 +++- scripts/fetch_core.py | 6 ++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 9a67521b..1d400647 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -30,10 +30,14 @@ def install(reinstall=False) -> None: with importlib.resources.path("pretext.resources", "core.zip") as static_zip: with zipfile.ZipFile(static_zip, "r") as zip: zip.extractall(path=RESOURCE_BASE_PATH) - (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(RESOURCE_BASE_PATH / "core") + (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( + RESOURCE_BASE_PATH / "core" + ) log.info("Installing templates") with importlib.resources.path("pretext.resources", "templates.zip") as static_zip: with zipfile.ZipFile(static_zip, "r") as zip: zip.extractall(path=RESOURCE_BASE_PATH) - (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename(RESOURCE_BASE_PATH / "core") + (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( + RESOURCE_BASE_PATH / "core" + ) diff --git a/scripts/bundle_resources.py b/scripts/bundle_resources.py index e705d124..4d467c4c 100644 --- a/scripts/bundle_resources.py +++ b/scripts/bundle_resources.py @@ -3,7 +3,9 @@ def main() -> None: - shutil.make_archive(str(Path("pretext") / "resources" / "templates"), 'zip', Path("templates")) + shutil.make_archive( + str(Path("pretext") / "resources" / "templates"), "zip", Path("templates") + ) print("Templates successfully zipped.") # TODO: incorporate in pelican branch # shutil.make_archive(str(Path("pretext") / "resources" / "pelican"), 'zip', Path("pelican")) diff --git a/scripts/fetch_core.py b/scripts/fetch_core.py index 50d1a030..778bb3b5 100644 --- a/scripts/fetch_core.py +++ b/scripts/fetch_core.py @@ -15,7 +15,7 @@ def main() -> None: r = requests.get( f"https://github.com/PreTeXtBook/pretext/archive/{CORE_COMMIT}.zip" ) - with open(core_zip_path, 'wb') as f: + with open(core_zip_path, "wb") as f: f.write(r.content) with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: with zipfile.ZipFile(core_zip_path) as archive: @@ -24,7 +24,9 @@ def main() -> None: path=tmpdirname, ) assert Path(pretext_py).exists() - shutil.copyfile(Path(pretext_py), Path("pretext").resolve() / "core" / "pretext.py") + shutil.copyfile( + Path(pretext_py), Path("pretext").resolve() / "core" / "pretext.py" + ) print("Successfully updated core PreTeXtBook/pretext resources from GitHub.") From f3fd01c44bc723b77906882de299d58ca6f62176 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:00:18 +0000 Subject: [PATCH 04/21] import bundle_resources tweak --- scripts/build_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_package.py b/scripts/build_package.py index 2b6710ac..6c931212 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -1,6 +1,6 @@ import subprocess import fetch_core -import scripts.bundle_resources as bundle_resources +import bundle_resources def main() -> None: From 9eff7050a14a5451b7753d68887224c8037fe196 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:08:16 +0000 Subject: [PATCH 05/21] remove deprecated templates module --- pretext/project/__init__.py | 6 +++--- pretext/resources/__init__.py | 4 ---- tests/test_project.py | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index ca93cbe7..2b696a29 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -24,7 +24,7 @@ from .. import codechat from .. import utils from .. import types as pt # PreTeXt types -from .. import templates +from ..resources import RESOURCE_BASE_PATH from .. import VERSION @@ -286,7 +286,7 @@ def post_validate(self) -> None: if not self.publication_abspath().exists(): # ... then use the CLI's built-in template file. # TODO: this is wrong, since the returned path is only valid inside the context manager. Instead, need to enter the context here, then exit it when this class is deleted (also problematic). - with templates.resource_path("publication.ptx") as self.publication: + with RESOURCE_BASE_PATH / "templates" / "publication.ptx" as self.publication: pass # Otherwise, verify that the provided publication file exists. TODO: It is silly to check that all publication files exist. We warn when they don't. If the target we are calling has a non-existent publication file, then that error will be caught anyway. else: @@ -1441,7 +1441,7 @@ def generate_boilerplate( f"Your existing {resource} file has been backed up at {backup_resource_path}." ) if resource != "requirements.txt": - with templates.resource_path(resource) as resource_path: + with RESOURCE_BASE_PATH / "templates" / resource as resource_path: if ( not project_resource_path.exists() or resource_path.read_text() diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 1d400647..7982306e 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -12,10 +12,6 @@ RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION -def path(resource_type: t.Literal["core", "templates"]) -> Path: - return Path(RESOURCE_BASE_PATH / resource_type) - - def install(reinstall=False) -> None: if RESOURCE_BASE_PATH.exists(): if reinstall: diff --git a/tests/test_project.py b/tests/test_project.py index 40b16003..8159d89d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -8,8 +8,8 @@ import pytest from pretext import project as pr -from pretext import templates from pretext import utils +from pretext.resources import RESOURCE_BASE_PATH from .common import DEMO_MAPPING, check_installed @@ -36,8 +36,6 @@ def test_defaults(tmp_path: Path) -> None: project = pr.Project(ptx_version="2") for t in ts: ts_dict[t[0]] = project.new_target(*t) - with templates.resource_path("publication.ptx") as pub_path: - pass assert project._path == Path.cwd() / Path("project.ptx") assert project.source == Path("source") assert project.publication == Path("publication") @@ -51,7 +49,7 @@ def test_defaults(tmp_path: Path) -> None: assert target.name == name assert target.format == format assert target.source == Path("main.ptx") - assert target.publication == pub_path + assert target.publication == RESOURCE_BASE_PATH / "templates" / "publication.ptx" assert target.output_dir == Path(name) assert target.deploy_dir is None assert target.xsl is None From 5995320dd72ba1d1f86ba5f810b9662a8fc343a6 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:09:37 +0000 Subject: [PATCH 06/21] formatting --- tests/test_project.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_project.py b/tests/test_project.py index 8159d89d..ab70db4b 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -49,7 +49,10 @@ def test_defaults(tmp_path: Path) -> None: assert target.name == name assert target.format == format assert target.source == Path("main.ptx") - assert target.publication == RESOURCE_BASE_PATH / "templates" / "publication.ptx" + assert ( + target.publication + == RESOURCE_BASE_PATH / "templates" / "publication.ptx" + ) assert target.output_dir == Path(name) assert target.deploy_dir is None assert target.xsl is None From 5b47e5b3b84a88a5376a02cb8a97c37ef8691e29 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:11:53 +0000 Subject: [PATCH 07/21] lint --- pretext/resources/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 7982306e..96af4d3b 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -2,7 +2,6 @@ import logging from pathlib import Path import shutil -import typing as t import zipfile from .. import VERSION, CORE_COMMIT From 147da6c4525f8c10bcfba3618fe47a54d003ca33 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:14:46 +0000 Subject: [PATCH 08/21] remove tests for deprecated templates module --- tests/test_templates.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 tests/test_templates.py diff --git a/tests/test_templates.py b/tests/test_templates.py deleted file mode 100644 index efafb900..00000000 --- a/tests/test_templates.py +++ /dev/null @@ -1,24 +0,0 @@ -from pretext.templates import resource_path - - -def test_resource_path() -> None: - resources = [ - ".devcontainer.json", - ".gitignore", - "article.zip", - "book.zip", - "codechat_config.yaml", - "demo.zip", - "hello.zip", - "project.ptx", - "publication.ptx", - "slideshow.zip", - ] - for filename in resources: - with resource_path(filename) as path: - assert path.name == filename - try: - with resource_path("does-not-exist.foo-bar") as path: - assert False - except FileNotFoundError: - assert True From 48d3971197c014ad9518f1fc58e7da173f8df046 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:24:14 +0000 Subject: [PATCH 09/21] clean up templates imports --- pretext/cli.py | 4 ++-- pretext/resources/__init__.py | 2 +- pretext/utils.py | 4 ++-- scripts/symlink_core.py | 20 ++++++++------------ 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pretext/cli.py b/pretext/cli.py index 6645377d..d13c583f 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -25,7 +25,7 @@ from . import ( utils, - templates, + resources, core, constants, plastex, @@ -296,7 +296,7 @@ def new(template: str, directory: Path, url_template: str) -> None: r = requests.get(url_template) archive = zipfile.ZipFile(io.BytesIO(r.content)) else: - with templates.resource_path(f"{template}.zip") as template_path: + with resources.RESOURCE_BASE_PATH / "templates" / f"{template}.zip" as template_path: archive = zipfile.ZipFile(template_path) # find (first) project.ptx to use as root of template filenames = [Path(filepath).name for filepath in archive.namelist()] diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 96af4d3b..6d390c76 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -11,7 +11,7 @@ RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION -def install(reinstall=False) -> None: +def install(reinstall: bool = False) -> None: if RESOURCE_BASE_PATH.exists(): if reinstall: log.info(f"Deleting existing resources at {RESOURCE_BASE_PATH}") diff --git a/pretext/utils.py b/pretext/utils.py index cb93278c..521cd234 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -19,7 +19,7 @@ from typing import Any, cast, List, Optional -from . import core, templates, constants +from . import core, constants, resources # Get access to logger log = logging.getLogger("ptxlogger") @@ -94,7 +94,7 @@ def project_xml(dirpath: t.Optional[Path] = None) -> _ElementTree: dirpath = Path() # current directory pp = project_path(dirpath) if pp is None: - with templates.resource_path("project.ptx") as project_manifest: + with resources.RESOURCE_BASE_PATH / "templates" / "project.ptx" as project_manifest: return ET.parse(project_manifest) else: project_manifest = pp / "project.ptx" diff --git a/scripts/symlink_core.py b/scripts/symlink_core.py index 6550299b..4f9b3f19 100644 --- a/scripts/symlink_core.py +++ b/scripts/symlink_core.py @@ -1,21 +1,17 @@ +import shutil import sys from pathlib import Path -from remove_path import remove_path -import pretext.core.resources +import pretext.resources # This will redirect the static resources for pretext, including the core python script, xsl, css, etc to a local directory of your choosing that contains the clone of the pretext repository. This is useful for development purposes, as it allows you to make changes to the core python script and test with the CLI as you normally would. def main(core_path: Path = Path("../pretext")) -> None: - for subdir in ["xsl", "schema", "script", "css", "js", "js_lib"]: - original_path = (core_path / subdir).resolve() - link_path = pretext.core.resources.path(subdir) - remove_path(link_path) - link_path.symlink_to(original_path) - original_path = (core_path / "pretext" / "pretext.py").resolve() - link_path = Path("pretext") / "core" / "pretext.py" - remove_path(link_path) - link_path.symlink_to(original_path) - + link_path = pretext.resources.RESOURCE_BASE_PATH / "core" + if link_path.is_dir(): + shutil.rmtree(link_path) + else: + link_path.unlink() + link_path.symlink_to(core_path) print(f"Linked local core pretext directory `{core_path}`") From 3bfc8abe2d2dc46142b285941b28d033580e14bc Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:52:32 +0000 Subject: [PATCH 10/21] fix set_ptx_path in core init --- pretext/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretext/core/__init__.py b/pretext/core/__init__.py index 9b876730..b51fa917 100644 --- a/pretext/core/__init__.py +++ b/pretext/core/__init__.py @@ -10,7 +10,7 @@ from .. import resources from .. import CORE_COMMIT, VERSION -set_ptx_path(resources.path("core")) +set_ptx_path(resources.RESOURCE_BASE_PATH / "core") def cli_build_message() -> str: From 767b1113ad6d3f50ab4137afbf318739c0e48cf9 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 20:56:50 +0000 Subject: [PATCH 11/21] fix templates install --- pretext/resources/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 6d390c76..8804bf36 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -30,9 +30,7 @@ def install(reinstall: bool = False) -> None: ) log.info("Installing templates") + (RESOURCE_BASE_PATH / "templates").mkdir() with importlib.resources.path("pretext.resources", "templates.zip") as static_zip: with zipfile.ZipFile(static_zip, "r") as zip: - zip.extractall(path=RESOURCE_BASE_PATH) - (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( - RESOURCE_BASE_PATH / "core" - ) + zip.extractall(path=RESOURCE_BASE_PATH / "templates") From 4812bd34a1788ad97c2402e9e2d91f78696599b9 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 21:47:49 +0000 Subject: [PATCH 12/21] fix cli new --- pretext/cli.py | 54 ++++++++++++++--------------------- pretext/core/__init__.py | 2 +- pretext/project/__init__.py | 6 ++-- pretext/resources/__init__.py | 29 ++++++++++++------- pretext/utils.py | 2 +- scripts/symlink_core.py | 2 +- tests/test_project.py | 4 +-- 7 files changed, 48 insertions(+), 51 deletions(-) diff --git a/pretext/cli.py b/pretext/cli.py index d13c583f..4becc8a2 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -284,47 +284,37 @@ def new(template: str, directory: Path, url_template: str) -> None: """ directory_fullpath = Path(directory).resolve() if utils.project_path(directory_fullpath) is not None: - log.warning( + log.error( f"A project already exists in `{utils.project_path(directory_fullpath)}`." ) - log.warning("No new project will be generated.") + log.error("No new project will be generated.") return log.info( - f"Generating new PreTeXt project in `{directory_fullpath}` using `{template}` template." + f"Generating new PreTeXt project in `{directory_fullpath}`" ) + directory_fullpath.mkdir(exist_ok=True) if url_template is not None: + log.info(f"Using template at `{url_template}`") + # get project and extract to directory r = requests.get(url_template) archive = zipfile.ZipFile(io.BytesIO(r.content)) + with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: + archive.extractall(tmpdirname) + content_path = [Path(tmpdirname) / i for i in os.listdir(tmpdirname)][0] + shutil.copytree(content_path, directory_fullpath, dirs_exist_ok=True) else: - with resources.RESOURCE_BASE_PATH / "templates" / f"{template}.zip" as template_path: - archive = zipfile.ZipFile(template_path) - # find (first) project.ptx to use as root of template - filenames = [Path(filepath).name for filepath in archive.namelist()] - project_ptx_index = filenames.index("project.ptx") - project_ptx_path = Path(archive.namelist()[project_ptx_index]) - project_dir_path = project_ptx_path.parent - with tempfile.TemporaryDirectory(prefix="pretext_") as tmpdirname: - temp_path = Path(tmpdirname) / "new-project" - temp_path.mkdir() - for filepath in [ - filepath - for filepath in archive.namelist() - if project_dir_path in Path(filepath).parents - ]: - archive.extract(filepath, path=temp_path) - tmpsubdirname = temp_path / project_dir_path - shutil.copytree(tmpsubdirname, directory_fullpath, dirs_exist_ok=True) - # generate remaining boilerplate like requirements.txt - project = Project.parse(directory_fullpath) - project.generate_boilerplate(update_requirements=True) - if len(project.targets) == 0: - log.warning("The generated project has no targets!") - else: - target = project.targets[0] - log.info(f"Success! Open `{target.source_abspath()}` to edit your document") - log.info( - f"Then try to `pretext build` and `pretext view` from within `{directory_fullpath}`." - ) + log.info(f"Using `{template}` template.") + # copy project from installed resources + with resources.resource_base_path() / "templates" / f"{template}" as template_path: + shutil.copytree(template_path, directory_fullpath, dirs_exist_ok=True) + # generate missing boilerplate + with utils.working_directory(directory_fullpath): + project_path = utils.project_path() + if project_path is None: + project = Project() + else: + project = Project.parse(project_path) + project.generate_boilerplate(update_requirements=True) # pretext init diff --git a/pretext/core/__init__.py b/pretext/core/__init__.py index b51fa917..c721554c 100644 --- a/pretext/core/__init__.py +++ b/pretext/core/__init__.py @@ -10,7 +10,7 @@ from .. import resources from .. import CORE_COMMIT, VERSION -set_ptx_path(resources.RESOURCE_BASE_PATH / "core") +set_ptx_path(resources.resource_base_path() / "core") def cli_build_message() -> str: diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 2b696a29..c8060002 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -24,7 +24,7 @@ from .. import codechat from .. import utils from .. import types as pt # PreTeXt types -from ..resources import RESOURCE_BASE_PATH +from ..resources import resource_base_path from .. import VERSION @@ -286,7 +286,7 @@ def post_validate(self) -> None: if not self.publication_abspath().exists(): # ... then use the CLI's built-in template file. # TODO: this is wrong, since the returned path is only valid inside the context manager. Instead, need to enter the context here, then exit it when this class is deleted (also problematic). - with RESOURCE_BASE_PATH / "templates" / "publication.ptx" as self.publication: + with resource_base_path() / "templates" / "publication.ptx" as self.publication: pass # Otherwise, verify that the provided publication file exists. TODO: It is silly to check that all publication files exist. We warn when they don't. If the target we are calling has a non-existent publication file, then that error will be caught anyway. else: @@ -1441,7 +1441,7 @@ def generate_boilerplate( f"Your existing {resource} file has been backed up at {backup_resource_path}." ) if resource != "requirements.txt": - with RESOURCE_BASE_PATH / "templates" / resource as resource_path: + with resource_base_path() / "templates" / resource as resource_path: if ( not project_resource_path.exists() or resource_path.read_text() diff --git a/pretext/resources/__init__.py b/pretext/resources/__init__.py index 8804bf36..0c271990 100644 --- a/pretext/resources/__init__.py +++ b/pretext/resources/__init__.py @@ -8,29 +8,36 @@ log = logging.getLogger("ptxlogger") -RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION +_RESOURCE_BASE_PATH = Path.home() / ".ptx" / VERSION def install(reinstall: bool = False) -> None: - if RESOURCE_BASE_PATH.exists(): + if _RESOURCE_BASE_PATH.exists(): if reinstall: - log.info(f"Deleting existing resources at {RESOURCE_BASE_PATH}") - shutil.rmtree(RESOURCE_BASE_PATH) + log.info(f"Deleting existing resources at {_RESOURCE_BASE_PATH}") + shutil.rmtree(_RESOURCE_BASE_PATH) else: - log.warning(f"Resources are already installed at {RESOURCE_BASE_PATH}") + log.warning(f"Resources are already installed at {_RESOURCE_BASE_PATH}") return - RESOURCE_BASE_PATH.mkdir(parents=True) + _RESOURCE_BASE_PATH.mkdir(parents=True) log.info("Installing core resources") with importlib.resources.path("pretext.resources", "core.zip") as static_zip: with zipfile.ZipFile(static_zip, "r") as zip: - zip.extractall(path=RESOURCE_BASE_PATH) - (RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( - RESOURCE_BASE_PATH / "core" + zip.extractall(path=_RESOURCE_BASE_PATH) + (_RESOURCE_BASE_PATH / f"pretext-{CORE_COMMIT}").rename( + _RESOURCE_BASE_PATH / "core" ) log.info("Installing templates") - (RESOURCE_BASE_PATH / "templates").mkdir() + (_RESOURCE_BASE_PATH / "templates").mkdir() with importlib.resources.path("pretext.resources", "templates.zip") as static_zip: with zipfile.ZipFile(static_zip, "r") as zip: - zip.extractall(path=RESOURCE_BASE_PATH / "templates") + zip.extractall(path=_RESOURCE_BASE_PATH / "templates") + + +def resource_base_path() -> Path: + if not _RESOURCE_BASE_PATH.exists(): + log.info(f"Installing resources to {_RESOURCE_BASE_PATH}") + install() + return _RESOURCE_BASE_PATH diff --git a/pretext/utils.py b/pretext/utils.py index 521cd234..95d35233 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -94,7 +94,7 @@ def project_xml(dirpath: t.Optional[Path] = None) -> _ElementTree: dirpath = Path() # current directory pp = project_path(dirpath) if pp is None: - with resources.RESOURCE_BASE_PATH / "templates" / "project.ptx" as project_manifest: + with resources.resource_base_path() / "templates" / "project.ptx" as project_manifest: return ET.parse(project_manifest) else: project_manifest = pp / "project.ptx" diff --git a/scripts/symlink_core.py b/scripts/symlink_core.py index 4f9b3f19..7617108a 100644 --- a/scripts/symlink_core.py +++ b/scripts/symlink_core.py @@ -6,7 +6,7 @@ # This will redirect the static resources for pretext, including the core python script, xsl, css, etc to a local directory of your choosing that contains the clone of the pretext repository. This is useful for development purposes, as it allows you to make changes to the core python script and test with the CLI as you normally would. def main(core_path: Path = Path("../pretext")) -> None: - link_path = pretext.resources.RESOURCE_BASE_PATH / "core" + link_path = pretext.resources.resource_base_path() / "core" if link_path.is_dir(): shutil.rmtree(link_path) else: diff --git a/tests/test_project.py b/tests/test_project.py index ab70db4b..9a9e59f1 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -9,7 +9,7 @@ from pretext import project as pr from pretext import utils -from pretext.resources import RESOURCE_BASE_PATH +from pretext.resources import resource_base_path from .common import DEMO_MAPPING, check_installed @@ -51,7 +51,7 @@ def test_defaults(tmp_path: Path) -> None: assert target.source == Path("main.ptx") assert ( target.publication - == RESOURCE_BASE_PATH / "templates" / "publication.ptx" + == resource_base_path() / "templates" / "publication.ptx" ) assert target.output_dir == Path(name) assert target.deploy_dir is None From cc78b15265fb23528ab4408c04873866b362306c Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 21:52:44 +0000 Subject: [PATCH 13/21] format --- pretext/cli.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pretext/cli.py b/pretext/cli.py index 4becc8a2..1c5c1003 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -289,9 +289,7 @@ def new(template: str, directory: Path, url_template: str) -> None: ) log.error("No new project will be generated.") return - log.info( - f"Generating new PreTeXt project in `{directory_fullpath}`" - ) + log.info(f"Generating new PreTeXt project in `{directory_fullpath}`") directory_fullpath.mkdir(exist_ok=True) if url_template is not None: log.info(f"Using template at `{url_template}`") From 1a976ec6208db9e9752b2c18b5f139ac5ce1103f Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 21:56:40 +0000 Subject: [PATCH 14/21] fix devscript alias --- pretext/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretext/cli.py b/pretext/cli.py index 1c5c1003..798d40b3 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -248,7 +248,7 @@ def devscript(args: List[str]) -> None: """ PY_CMD = sys.executable subprocess.run( - [PY_CMD, str(core.resources.path("pretext", "pretext"))] + list(args) + [PY_CMD, str(resources.resource_base_path() / "pretext" / "pretext")] + list(args) ) From cef7a4989a5ee1af8d00ace3bf24307ecebd38f1 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 21:59:39 +0000 Subject: [PATCH 15/21] fix devscript --- pretext/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pretext/cli.py b/pretext/cli.py index 798d40b3..fb7a84d6 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -248,7 +248,8 @@ def devscript(args: List[str]) -> None: """ PY_CMD = sys.executable subprocess.run( - [PY_CMD, str(resources.resource_base_path() / "pretext" / "pretext")] + list(args) + [PY_CMD, str(resources.resource_base_path() / "core" / "pretext" / "pretext")] + + list(args) ) From b787c72c8ebad78405969843af61fc2982f2b425 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:04:36 +0000 Subject: [PATCH 16/21] fix stale core.resources references --- pretext/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pretext/utils.py b/pretext/utils.py index 95d35233..0e6eeb12 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -173,7 +173,7 @@ def xml_syntax_is_valid(xmlfile: Path, root_tag: str = "pretext") -> bool: def xml_source_validates_against_schema(xmlfile: Path) -> bool: # get path to RelaxNG schema file: - schemarngfile = core.resources.path("schema", "pretext.rng") + schemarngfile = resources.resource_base_path() / "core" / "schema" / "pretext.rng" # Open schemafile for validation: relaxng = ET.RelaxNG(file=schemarngfile) @@ -295,7 +295,7 @@ def copy_custom_xsl(xsl_path: Path, output_dir: Path) -> None: log.debug(f"Copying all files in {xsl_dir} to {output_dir}") shutil.copytree(xsl_dir, output_dir, dirs_exist_ok=True) log.debug(f"Copying core XSL to {output_dir}/core") - shutil.copytree(core.resources.path("xsl"), output_dir / "core") + shutil.copytree(resources.resource_base_path() / "core" / "xsl", output_dir / "core") def check_executable(exec_name: str) -> Optional[str]: @@ -439,7 +439,7 @@ def show_target_hints( def npm_install() -> None: - with working_directory(core.resources.path("script", "mjsre")): + with working_directory(resources.resource_base_path() / "core" / "script" / "mjsre"): log.info("Attempting to install/update required node packages.") try: subprocess.run("npm install", shell=True) From 005d24dde7eee2a42533986ca332213c614f0148 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:06:24 +0000 Subject: [PATCH 17/21] formatting --- pretext/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pretext/utils.py b/pretext/utils.py index 0e6eeb12..0c3ebb5e 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -295,7 +295,9 @@ def copy_custom_xsl(xsl_path: Path, output_dir: Path) -> None: log.debug(f"Copying all files in {xsl_dir} to {output_dir}") shutil.copytree(xsl_dir, output_dir, dirs_exist_ok=True) log.debug(f"Copying core XSL to {output_dir}/core") - shutil.copytree(resources.resource_base_path() / "core" / "xsl", output_dir / "core") + shutil.copytree( + resources.resource_base_path() / "core" / "xsl", output_dir / "core" + ) def check_executable(exec_name: str) -> Optional[str]: @@ -439,7 +441,9 @@ def show_target_hints( def npm_install() -> None: - with working_directory(resources.resource_base_path() / "core" / "script" / "mjsre"): + with working_directory( + resources.resource_base_path() / "core" / "script" / "mjsre" + ): log.info("Attempting to install/update required node packages.") try: subprocess.run("npm install", shell=True) From 3be6a303b6eee01aaef978799a45746525b1b4d4 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:19:39 +0000 Subject: [PATCH 18/21] add sample article test --- tests/test_sample_article.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/test_sample_article.py diff --git a/tests/test_sample_article.py b/tests/test_sample_article.py new file mode 100644 index 00000000..d01c4503 --- /dev/null +++ b/tests/test_sample_article.py @@ -0,0 +1,20 @@ +import shutil +import pytest +from pathlib import Path +from pretext.project import Project +from pretext.resources import resource_base_path +import pretext.utils +from .common import check_installed + + +@pytest.mark.skipif( + not check_installed(["xelatex", "--version"]), + reason="Note: several tests are skipped, since xelatex wasn't installed.", +) +def test_sample_article(tmp_path: Path) -> None: + prj_path = tmp_path / "sample" + shutil.copytree(resource_base_path() / "core" / "examples" / "sample-article", prj_path) + with pretext.utils.working_directory(prj_path): + project = Project.parse() + t = project.get_target() + t.build() From 85a264ca0fae7775c5dfd10a9719b6c871205c8c Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:21:09 +0000 Subject: [PATCH 19/21] formatting --- tests/test_sample_article.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_sample_article.py b/tests/test_sample_article.py index d01c4503..edd101be 100644 --- a/tests/test_sample_article.py +++ b/tests/test_sample_article.py @@ -13,7 +13,9 @@ ) def test_sample_article(tmp_path: Path) -> None: prj_path = tmp_path / "sample" - shutil.copytree(resource_base_path() / "core" / "examples" / "sample-article", prj_path) + shutil.copytree( + resource_base_path() / "core" / "examples" / "sample-article", prj_path + ) with pretext.utils.working_directory(prj_path): project = Project.parse() t = project.get_target() From cfd24ff0e7a8e5c7314f96f786fcaab717649b50 Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:38:17 +0000 Subject: [PATCH 20/21] more accurate sample article checker --- tests/test_sample_article.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_sample_article.py b/tests/test_sample_article.py index edd101be..080c58ff 100644 --- a/tests/test_sample_article.py +++ b/tests/test_sample_article.py @@ -1,6 +1,7 @@ import shutil import pytest from pathlib import Path +import errorhandler # type: ignore from pretext.project import Project from pretext.resources import resource_base_path import pretext.utils @@ -12,6 +13,7 @@ reason="Note: several tests are skipped, since xelatex wasn't installed.", ) def test_sample_article(tmp_path: Path) -> None: + error_checker = errorhandler.ErrorHandler(logger="ptxlogger") prj_path = tmp_path / "sample" shutil.copytree( resource_base_path() / "core" / "examples" / "sample-article", prj_path @@ -20,3 +22,4 @@ def test_sample_article(tmp_path: Path) -> None: project = Project.parse() t = project.get_target() t.build() + assert not error_checker.fired From 9202d22b81add02073c496527eb4fe2b424f518c Mon Sep 17 00:00:00 2001 From: Steven Clontz Date: Thu, 20 Jun 2024 22:52:56 +0000 Subject: [PATCH 21/21] update core commit to get working sample article --- pretext/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretext/__init__.py b/pretext/__init__.py index fa235a39..f35220b7 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -21,7 +21,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "8e9aac01da8fdab44c6c9ded4f30d9d7227d015b" +CORE_COMMIT = "ac1ca3ca67c9512059afd6fd37a714a7fc988a5d" def activate() -> None: