From c31cdd135869a995dce7c46a324a5457944c2567 Mon Sep 17 00:00:00 2001 From: Brendan Kiu Date: Tue, 21 May 2024 16:03:21 -0400 Subject: [PATCH 1/5] Adding a command to allow converting a project for use offline --- src/pyscript/_generator.py | 15 ++- src/pyscript/cli.py | 2 +- src/pyscript/plugins/convert_offline.py | 171 ++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 src/pyscript/plugins/convert_offline.py diff --git a/src/pyscript/_generator.py b/src/pyscript/_generator.py index 0489f84..efae7f4 100644 --- a/src/pyscript/_generator.py +++ b/src/pyscript/_generator.py @@ -161,17 +161,22 @@ def create_project( def _get_latest_pyscript_version() -> str: """Get the latest version of PyScript from GitHub.""" - url = "https://api.github.com/repos/pyscript/pyscript/releases/latest" + return _get_latest_repo_version("pyscript", "pyscript", LATEST_PYSCRIPT_VERSION) + + +def _get_latest_repo_version(gh_user: str, gh_repo: str, default: str): + """Get the latest version of given repo from GitHub.""" + url = f"https://api.github.com/repos/{gh_user}/{gh_repo}/releases/latest" try: response = requests.get(url) if not response.ok: - pyscript_version = LATEST_PYSCRIPT_VERSION + version = default else: data = response.json() - pyscript_version = data["tag_name"] + version = data["tag_name"] except Exception: - pyscript_version = LATEST_PYSCRIPT_VERSION + version = default - return pyscript_version + return version diff --git a/src/pyscript/cli.py b/src/pyscript/cli.py index 9757ca3..3eca504 100644 --- a/src/pyscript/cli.py +++ b/src/pyscript/cli.py @@ -8,7 +8,7 @@ from pyscript import __version__, app, console, plugins, typer from pyscript.plugins import hookspecs -DEFAULT_PLUGINS = ["create", "run"] +DEFAULT_PLUGINS = ["create", "run", "convert_offline"] def ok(msg: str = ""): diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py new file mode 100644 index 0000000..582f729 --- /dev/null +++ b/src/pyscript/plugins/convert_offline.py @@ -0,0 +1,171 @@ +import json +import os +import re +import tarfile +import tempfile +import toml + +from pathlib import Path +from typing import Optional + +import requests +import typer + +from pyscript import app, cli, plugins +from pyscript._generator import ( + _get_latest_pyscript_version, + _get_latest_repo_version, + save_config_file, +) + +@app.command() +def convert_offline( + path: str = typer.Argument( + None, help="Path to pyscript project to convert for offline usage" + ), + config_files: str = typer.Option( + "pyscript.toml", + "--config-files", + help="Comma-separated list of config files" + ), + interpreter: str = typer.Option( + "pyodide", + "--interpreter", + help="Choose which interpreter to configure. Choices are 'pyodide' or 'micropython'" + ), + download_full_pyodide: bool = typer.Option( + False, + "--download-full-pyodide", + help="Download the 200MB+ pyodide libraries instead of just required interpreter" + ) +): + """ + Takes an existing pyscript app and converts it for offline usage + """ + app_path = Path(path) + PYSCRIPT_TAR_URL_BASE = "https://pyscript.net/releases/{pyscript_version}/release.tar" + PYODIDE_TAR_URL_BASE = "https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" + MPY_BASE_URL = "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" + remote_pyscript_pattern = re.compile(r'https://pyscript.net/releases/[\d.]+/') + + if interpreter not in ("pyodide", "micropython"): + raise cli.Abort("Interpreter must be one of 'pyodide' or 'micropython'") + + # Get first app configuration to pull pyscript version + config_files_list = config_files.split(",") + config_path = app_path / config_files_list[0] + config = _get_config(config_path) + + # Get the required pyscript version based on config + pyscript_version = config.get('version', 'latest') + if pyscript_version == 'latest': + pyscript_version = _get_latest_pyscript_version() + + pyscript_tar_url = PYSCRIPT_TAR_URL_BASE.format(pyscript_version=pyscript_version) + pyscript_files_dir = app_path / "pyscript" + + # Download and extract pyscript files + print("Downloading pyscript files...") + _download_and_extract_tarfile(pyscript_tar_url, pyscript_files_dir) + print("Downloading and extraction of pyscript files successful.") + + # Download and extract pyodide files + print("Downloading pyodide files...") + pyodide_version = _get_latest_repo_version("pyodide", "pyodide", None) + if not pyodide_version: + raise cli.Abort("Unable to retrieve latest pyodide version from Github") + + # Only download the 200MB+ pyodide libraries if required, else just download + # the core + pyodide_tar_name = "pyodide" if download_full_pyodide else "pyodide-core" + pyodide_tar_url = PYODIDE_TAR_URL_BASE.format( + pyodide_version=pyodide_version, + pyodide_tar_name=pyodide_tar_name + ) + _download_and_extract_tarfile(pyodide_tar_url, app_path) + print("Downloading and extraction of pyodide files successful.") + + # Download Micropython files + print("Downloading micropython files...") + mpy_path = app_path / "micropython" + mpy_path.mkdir(exist_ok=True) + files = ("micropython.mjs", "micropython.wasm") + + for file in files: + target_path = mpy_path / file + url = MPY_BASE_URL + file + + # wasm file is bytes format, mjs is text + response = requests.get(url) + if 'wasm' in file: + with open(target_path, 'wb') as fp: + fp.write(response.content) + else: + with open(target_path, 'w') as fp: + fp.write(response.text) + print("Downloading of micropython files sucessful") + + # Finding all HTML files + html_files = [] + for dirpath, dirnames, filenames in app_path.walk(): + html_files.extend([dirpath / f for f in filenames if f.endswith(".html")]) + + # Replace remote resources with freshly downloaded resources + # Also for old config format to warn user + old_config_pattern = re.compile(r'py-config>') + found_old_config = False + + for filepath in html_files: + with open(filepath, 'r') as fpi: + content = fpi.read() + + if remote_pyscript_pattern.search(content): + new_content = remote_pyscript_pattern.sub("/pyscript/", content) + with open(filepath, 'w') as fpo: + fpo.write(new_content) + print(f"Updated {filepath}") + + found_old_config = found_old_config or old_config_pattern.search(content) + + # Add/replace interpreter with downloaded interpreter + for config_file in config_files_list: + config_file_path = app_path / config_file + config = _get_config(config_file_path) + config['interpreter'] = f"/{interpreter}/{interpreter}.mjs" + save_config_file(config_file_path, config) + + print(f"Updated {config_file_path}") + + if found_old_config: + print("WARNING: and are not currently supported by this tool") + + + +def _download_and_extract_tarfile(remote_url: str, extract_dir: Path): + """Downloads the tarfile at `remote_url` and extracts it into `extract_dir` + + Params: + - remote_url(str): URL of the tarball, for example https://example.com/file.tar + - extract_dir(Path): directory to extract the tarball into + """ + with tempfile.TemporaryDirectory() as tempdirname: + tarfile_target = Path(tempdirname) / "temp.tar" + response = requests.get(remote_url, stream=True) + if response.status_code == 200: + with open(tarfile_target, "wb") as fp: + fp.write(response.raw.read()) + else: + raise cli.Abort(f"Unable to download required files. Please check your network connection") + + with tarfile.open(tarfile_target, 'r') as tfile: + tfile.extractall(path=extract_dir) + + +def _get_config(config_path: Path): + """Loads the configuration from the given Path""" + if "toml" in str(config_path): + return toml.load(config_path) + elif "json" in str(config_path): + with open(config_path, 'r') as fp: + return json.load(fp) + From e98c26b4a1823bf3e51fe7beed8f7227ee293bee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 20:13:40 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyscript/plugins/convert_offline.py | 71 +++++++++++++------------ 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py index 582f729..86b973c 100644 --- a/src/pyscript/plugins/convert_offline.py +++ b/src/pyscript/plugins/convert_offline.py @@ -3,50 +3,52 @@ import re import tarfile import tempfile -import toml - from pathlib import Path from typing import Optional import requests +import toml import typer from pyscript import app, cli, plugins from pyscript._generator import ( - _get_latest_pyscript_version, + _get_latest_pyscript_version, _get_latest_repo_version, save_config_file, ) + @app.command() def convert_offline( path: str = typer.Argument( None, help="Path to pyscript project to convert for offline usage" ), config_files: str = typer.Option( - "pyscript.toml", - "--config-files", - help="Comma-separated list of config files" + "pyscript.toml", "--config-files", help="Comma-separated list of config files" ), interpreter: str = typer.Option( "pyodide", "--interpreter", - help="Choose which interpreter to configure. Choices are 'pyodide' or 'micropython'" + help="Choose which interpreter to configure. Choices are 'pyodide' or 'micropython'", ), download_full_pyodide: bool = typer.Option( False, "--download-full-pyodide", - help="Download the 200MB+ pyodide libraries instead of just required interpreter" - ) + help="Download the 200MB+ pyodide libraries instead of just required interpreter", + ), ): """ Takes an existing pyscript app and converts it for offline usage """ app_path = Path(path) - PYSCRIPT_TAR_URL_BASE = "https://pyscript.net/releases/{pyscript_version}/release.tar" + PYSCRIPT_TAR_URL_BASE = ( + "https://pyscript.net/releases/{pyscript_version}/release.tar" + ) PYODIDE_TAR_URL_BASE = "https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" - MPY_BASE_URL = "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" - remote_pyscript_pattern = re.compile(r'https://pyscript.net/releases/[\d.]+/') + MPY_BASE_URL = ( + "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" + ) + remote_pyscript_pattern = re.compile(r"https://pyscript.net/releases/[\d.]+/") if interpreter not in ("pyodide", "micropython"): raise cli.Abort("Interpreter must be one of 'pyodide' or 'micropython'") @@ -57,9 +59,9 @@ def convert_offline( config = _get_config(config_path) # Get the required pyscript version based on config - pyscript_version = config.get('version', 'latest') - if pyscript_version == 'latest': - pyscript_version = _get_latest_pyscript_version() + pyscript_version = config.get("version", "latest") + if pyscript_version == "latest": + pyscript_version = _get_latest_pyscript_version() pyscript_tar_url = PYSCRIPT_TAR_URL_BASE.format(pyscript_version=pyscript_version) pyscript_files_dir = app_path / "pyscript" @@ -79,8 +81,7 @@ def convert_offline( # the core pyodide_tar_name = "pyodide" if download_full_pyodide else "pyodide-core" pyodide_tar_url = PYODIDE_TAR_URL_BASE.format( - pyodide_version=pyodide_version, - pyodide_tar_name=pyodide_tar_name + pyodide_version=pyodide_version, pyodide_tar_name=pyodide_tar_name ) _download_and_extract_tarfile(pyodide_tar_url, app_path) print("Downloading and extraction of pyodide files successful.") @@ -96,15 +97,15 @@ def convert_offline( url = MPY_BASE_URL + file # wasm file is bytes format, mjs is text - response = requests.get(url) - if 'wasm' in file: - with open(target_path, 'wb') as fp: + response = requests.get(url) + if "wasm" in file: + with open(target_path, "wb") as fp: fp.write(response.content) else: - with open(target_path, 'w') as fp: + with open(target_path, "w") as fp: fp.write(response.text) print("Downloading of micropython files sucessful") - + # Finding all HTML files html_files = [] for dirpath, dirnames, filenames in app_path.walk(): @@ -112,16 +113,16 @@ def convert_offline( # Replace remote resources with freshly downloaded resources # Also for old config format to warn user - old_config_pattern = re.compile(r'py-config>') + old_config_pattern = re.compile(r"py-config>") found_old_config = False for filepath in html_files: - with open(filepath, 'r') as fpi: + with open(filepath) as fpi: content = fpi.read() if remote_pyscript_pattern.search(content): new_content = remote_pyscript_pattern.sub("/pyscript/", content) - with open(filepath, 'w') as fpo: + with open(filepath, "w") as fpo: fpo.write(new_content) print(f"Updated {filepath}") @@ -131,19 +132,20 @@ def convert_offline( for config_file in config_files_list: config_file_path = app_path / config_file config = _get_config(config_file_path) - config['interpreter'] = f"/{interpreter}/{interpreter}.mjs" + config["interpreter"] = f"/{interpreter}/{interpreter}.mjs" save_config_file(config_file_path, config) print(f"Updated {config_file_path}") if found_old_config: - print("WARNING: and are not currently supported by this tool") - + print( + "WARNING: and are not currently supported by this tool" + ) def _download_and_extract_tarfile(remote_url: str, extract_dir: Path): """Downloads the tarfile at `remote_url` and extracts it into `extract_dir` - + Params: - remote_url(str): URL of the tarball, for example https://example.com/file.tar - extract_dir(Path): directory to extract the tarball into @@ -155,10 +157,12 @@ def _download_and_extract_tarfile(remote_url: str, extract_dir: Path): with open(tarfile_target, "wb") as fp: fp.write(response.raw.read()) else: - raise cli.Abort(f"Unable to download required files. Please check your network connection") + raise cli.Abort( + f"Unable to download required files. Please check your network connection" + ) - with tarfile.open(tarfile_target, 'r') as tfile: - tfile.extractall(path=extract_dir) + with tarfile.open(tarfile_target, "r") as tfile: + tfile.extractall(path=extract_dir) def _get_config(config_path: Path): @@ -166,6 +170,5 @@ def _get_config(config_path: Path): if "toml" in str(config_path): return toml.load(config_path) elif "json" in str(config_path): - with open(config_path, 'r') as fp: + with open(config_path) as fp: return json.load(fp) - From f4b385253292bf141960f7038c15d7a9c3c7916f Mon Sep 17 00:00:00 2001 From: Brendan Kiu Date: Wed, 22 May 2024 13:31:26 -0400 Subject: [PATCH 3/5] Fixing for mypy tests --- src/pyscript/plugins/convert_offline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py index 86b973c..86f7b4f 100644 --- a/src/pyscript/plugins/convert_offline.py +++ b/src/pyscript/plugins/convert_offline.py @@ -73,7 +73,7 @@ def convert_offline( # Download and extract pyodide files print("Downloading pyodide files...") - pyodide_version = _get_latest_repo_version("pyodide", "pyodide", None) + pyodide_version = _get_latest_repo_version("pyodide", "pyodide", "") if not pyodide_version: raise cli.Abort("Unable to retrieve latest pyodide version from Github") @@ -108,7 +108,8 @@ def convert_offline( # Finding all HTML files html_files = [] - for dirpath, dirnames, filenames in app_path.walk(): + for dirname, dirs, filenames in os.walk(app_path): + dirpath = Path(dirname) html_files.extend([dirpath / f for f in filenames if f.endswith(".html")]) # Replace remote resources with freshly downloaded resources @@ -126,7 +127,7 @@ def convert_offline( fpo.write(new_content) print(f"Updated {filepath}") - found_old_config = found_old_config or old_config_pattern.search(content) + found_old_config = found_old_config or bool(old_config_pattern.search(content)) # Add/replace interpreter with downloaded interpreter for config_file in config_files_list: From b005c080ba84491c3986c8bfffb6ff1942e752ba Mon Sep 17 00:00:00 2001 From: Brendan Kiu Date: Wed, 22 May 2024 13:43:44 -0400 Subject: [PATCH 4/5] Fixing for flake8 --- src/pyscript/plugins/convert_offline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py index 86f7b4f..735da4d 100644 --- a/src/pyscript/plugins/convert_offline.py +++ b/src/pyscript/plugins/convert_offline.py @@ -4,13 +4,12 @@ import tarfile import tempfile from pathlib import Path -from typing import Optional import requests import toml import typer -from pyscript import app, cli, plugins +from pyscript import app, cli from pyscript._generator import ( _get_latest_pyscript_version, _get_latest_repo_version, @@ -44,7 +43,8 @@ def convert_offline( PYSCRIPT_TAR_URL_BASE = ( "https://pyscript.net/releases/{pyscript_version}/release.tar" ) - PYODIDE_TAR_URL_BASE = "https://github.com/pyodide/pyodide/releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" + PYODIDE_TAR_URL_BASE = "https://github.com/pyodide/pyodide/" \ + "releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" MPY_BASE_URL = ( "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" ) @@ -159,7 +159,7 @@ def _download_and_extract_tarfile(remote_url: str, extract_dir: Path): fp.write(response.raw.read()) else: raise cli.Abort( - f"Unable to download required files. Please check your network connection" + "Unable to download required files. Please check your network connection" ) with tarfile.open(tarfile_target, "r") as tfile: From e24148bd0da8d2a6353e818debe3f040bada959c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 17:43:59 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/pyscript/plugins/convert_offline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pyscript/plugins/convert_offline.py b/src/pyscript/plugins/convert_offline.py index 735da4d..c4cbc56 100644 --- a/src/pyscript/plugins/convert_offline.py +++ b/src/pyscript/plugins/convert_offline.py @@ -43,8 +43,10 @@ def convert_offline( PYSCRIPT_TAR_URL_BASE = ( "https://pyscript.net/releases/{pyscript_version}/release.tar" ) - PYODIDE_TAR_URL_BASE = "https://github.com/pyodide/pyodide/" \ + PYODIDE_TAR_URL_BASE = ( + "https://github.com/pyodide/pyodide/" "releases/download/{pyodide_version}/{pyodide_tar_name}-{pyodide_version}.tar.bz2" + ) MPY_BASE_URL = ( "https://cdn.jsdelivr.net/npm/@micropython/micropython-webassembly-pyscript/" )