Skip to content

Commit 28d35bd

Browse files
martinRenoumartinRenou
authored
Merge pull request #76 from trungleduc/conda-format
Support direct download for no-arch packages
2 parents b27943b + b7dcff8 commit 28d35bd

File tree

10 files changed

+310
-18
lines changed

10 files changed

+310
-18
lines changed

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
-c https://repo.mamba.pm/emscripten-forge \
5454
-c https://repo.mamba.pm/conda-forge \
5555
--yes \
56-
python pybind11 nlohmann_json pybind11_json numpy "pytest==7.1.1" bzip2 sqlite zlib libffi exceptiongroup
56+
python pybind11 nlohmann_json pybind11_json numpy "pytest==7.1.1" bzip2 sqlite zlib zstd libffi exceptiongroup
5757
5858
5959
mkdir build

‎CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ set(PYJS_HEADERS
8383
include/pyjs/export_py_object.hpp
8484
include/pyjs/export_pyjs_module.hpp
8585
include/pyjs/untar.hpp
86+
include/pyjs/install_conda_file.hpp
8687
include/pyjs/inflate.hpp
8788
${CMAKE_CURRENT_BINARY_DIR}/pyjs_pre.js
8889
${CMAKE_CURRENT_BINARY_DIR}/pyjs_post.js
@@ -110,6 +111,7 @@ add_library(pyjs STATIC
110111
src/js_timestamp.cpp
111112
src/inflate.cpp
112113
src/untar.cpp
114+
src/install_conda_file.cpp
113115
${PYCPPSOURCES}
114116
)
115117

@@ -153,6 +155,7 @@ SET(PYTHON_UTIL_LIBS
153155
${CMAKE_INSTALL_PREFIX}/lib/libz.a
154156
${CMAKE_INSTALL_PREFIX}/lib/libsqlite3.a
155157
${CMAKE_INSTALL_PREFIX}/lib/libffi.a
158+
${CMAKE_INSTALL_PREFIX}/lib/libzstd.a
156159
)
157160

158161

‎build_mkdocs.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ if [ ! -d "$WASM_ENV_PREFIX" ]; then
2828
-c https://repo.mamba.pm/conda-forge \
2929
--yes \
3030
python pybind11 nlohmann_json pybind11_json numpy \
31-
bzip2 sqlite zlib libffi exceptiongroup \
31+
bzip2 sqlite zlib zstd libffi exceptiongroup \
3232
"xeus<4" "xeus-lite<2" xeus-python "xeus-javascript>=0.3.2" xtl "ipython=8.22.2=py311had7285e_1" "traitlets>=5.14.2"
3333

3434
else

‎include/pyjs/install_conda_file.hpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#pragma once
2+
3+
#include <emscripten/bind.h>
4+
5+
namespace pyjs
6+
{
7+
8+
em::val install_conda_file(const std::string& zstd_file_path,
9+
const std::string& working_dir,
10+
const std::string& path);
11+
12+
}

‎include/pyjs/pre_js/load_pkg.js‎

Lines changed: 111 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,69 @@ def _py_untar(tarball_path, target_dir):
6767
}
6868

6969

70+
Module["_unzip_from_python"] = function(tarball_path, target_dir) {
71+
Module.exec(`
72+
def _py_unzip(tarball_path, target_dir):
73+
import json
74+
from pathlib import Path
75+
import zipfile
76+
77+
target = Path(target_dir)
78+
target.mkdir(parents=True, exist_ok=True)
79+
pkg_file = {"name": "", "path": ""}
80+
with zipfile.ZipFile(tarball_path, mode="r") as archive:
81+
82+
for filename in archive.namelist():
83+
if filename.startswith("pkg-"):
84+
pkg_file["name"] = filename
85+
pkg_file["path"] = str(target / filename)
86+
archive.extract(filename, target_dir)
87+
break
88+
return json.dumps(pkg_file)
89+
90+
`)
91+
let extracted_file = Module.eval(`_py_unzip("${tarball_path}", "${target_dir}")`)
92+
93+
return JSON.parse(extracted_file)
94+
}
95+
96+
Module["_install_conda_file_from_python"] = function(tarball_path, target_dir) {
97+
Module.exec(`
98+
def _py_unbz2(tarball_path, target_dir):
99+
import json
100+
from pathlib import Path
101+
import tarfile
102+
import shutil
103+
import os
104+
import sys
105+
106+
target = Path(target_dir)
107+
prefix = Path(sys.prefix)
108+
try:
109+
with tarfile.open(tarball_path) as tar:
110+
tar.extractall(target_dir)
111+
112+
src = target / "site-packages"
113+
dest = prefix / "lib/python3.11/site-packages"
114+
shutil.copytree(src, dest, dirs_exist_ok=True)
115+
for folder in ["etc", "share"]:
116+
src = target / folder
117+
dest = prefix / folder
118+
if src.exists():
119+
shutil.copytree(src, dest, dirs_exist_ok=True)
120+
shutil.rmtree(target)
121+
except Exception as e:
122+
print("ERROR",e)
123+
raise e
124+
125+
return json.dumps([])
126+
127+
`)
128+
let extracted_file = Module.eval(`_py_unbz2("${tarball_path}", "${target_dir}")`)
129+
130+
return JSON.parse(extracted_file)
131+
}
132+
70133

71134

72135

@@ -108,20 +171,55 @@ Module["bootstrap_from_empack_packed_environment"] = async function
108171
pkg,
109172
verbose
110173
) {
111-
const package_url = pkg?.url ?? `${package_tarballs_root_url}/${pkg.filename}`;
112-
if (verbose) {
113-
console.log(`!!fetching pkg ${pkg.name} from ${package_url}`)
114-
}
115-
let byte_array = await fetchByteArray(package_url)
116-
const tarball_path = `/package_tarballs/${pkg.filename}`;
117-
Module.FS.writeFile(tarball_path, byte_array);
118-
if(verbose){
119-
console.log(`!!extract ${tarball_path} (${byte_array.length} bytes)`)
174+
const package_url =
175+
pkg?.url ?? `${package_tarballs_root_url}/${pkg.filename}`;
176+
if (verbose) {
177+
console.log(`!!fetching pkg ${pkg.name} from ${package_url}`);
178+
}
179+
let byte_array = await fetchByteArray(package_url);
180+
const tarball_path = `/package_tarballs/${pkg.filename}`;
181+
Module.FS.writeFile(tarball_path, byte_array);
182+
if (verbose) {
183+
console.log(
184+
`!!extract ${tarball_path} (${byte_array.length} bytes)`
185+
);
186+
}
187+
188+
if (verbose) {
189+
console.log("await python_is_ready_promise");
190+
}
191+
await python_is_ready_promise;
192+
193+
if (package_url.toLowerCase().endsWith(".conda")) {
194+
// Conda v2 packages
195+
if (verbose) {
196+
console.log(
197+
`!!extract conda package ${package_url} (${byte_array.length} bytes)`
198+
);
199+
}
200+
const dest = `/conda_packages/${pkg.name}`;
201+
const pkg_file = Module["_unzip_from_python"](
202+
tarball_path,
203+
dest
204+
);
205+
return Module._install_conda_file(pkg_file.path, dest, prefix);
206+
} else if (package_url.toLowerCase().endsWith(".tar.bz2")) {
207+
// Conda v1 packages
208+
if (verbose) {
209+
console.log(
210+
`!!extract conda package ${package_url} (${byte_array.length} bytes)`
211+
);
212+
}
213+
const dest = `/conda_packages/${pkg.name}`;
214+
return Module["_install_conda_file_from_python"](
215+
tarball_path,
216+
dest
217+
);
218+
} else {
219+
// Pre-relocated packages
220+
return Module["_untar_from_python"](tarball_path);
221+
}
120222
}
121-
if(verbose){console.log("await python_is_ready_promise");}
122-
await python_is_ready_promise;
123-
return Module["_untar_from_python"](tarball_path);
124-
}
125223

126224

127225
async function bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose) {

‎include/pyjs/untar.hpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
#pragma once
22

33
#include <emscripten/bind.h>
4+
#include <emscripten/val.h>
45

6+
namespace em = emscripten;
57
namespace pyjs{
6-
78
em::val untar(const std::string &tar_path, const std::string &path);
8-
9+
void untar_impl(FILE *a, const char *path, em::val & shared_libraraies);
910
}

‎src/convert.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
namespace py = pybind11;
1010
namespace em = emscripten;
1111

12+
1213
namespace pyjs
1314
{
1415
std::pair<em::val,bool> implicit_py_to_js(py::object& py_ret)

‎src/export_js_module.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <pyjs/export_py_object.hpp>
55
#include <pyjs/convert.hpp>
66
#include <pyjs/untar.hpp>
7+
#include <pyjs/install_conda_file.hpp>
78

89
#include <pybind11/embed.h>
910
#include <emscripten/bind.h>
@@ -105,6 +106,7 @@ namespace pyjs
105106

106107

107108
em::function("_untar", &untar);
109+
em::function("_install_conda_file", &install_conda_file);
108110
em::function("setenv", &set_env);
109111

110112
// py-object (proxy)

0 commit comments

Comments
 (0)