Commit 61d74fde authored by Paul Sokolovsky's avatar Paul Sokolovsky
Browse files

tools, unix: Replace upip tarball with just source files.

To make its inclusion as frozen modules in multiple ports less magic.
Ports are just expected to symlink 2 files into their scripts/modules
subdirs.

Unix port updated to use this and in general follow frozen modules setup
tested and tried on baremetal ports, where there's "scripts" predefined
dir (overridable with FROZEN_DIR make var), and a user just drops Python
files there.
parent bc4441af
import sys
import gc
import uos as os
import uerrno as errno
import ujson as json
import uzlib
import upip_utarfile as tarfile
gc.collect()
debug = False
install_path = None
cleanup_files = []
gzdict_sz = 16 + 15
file_buf = bytearray(512)
class NotFoundError(Exception):
pass
def op_split(path):
if path == "":
return ("", "")
r = path.rsplit("/", 1)
if len(r) == 1:
return ("", path)
head = r[0]
if not head:
head = "/"
return (head, r[1])
def op_basename(path):
return op_split(path)[1]
# Expects *file* name
def _makedirs(name, mode=0o777):
ret = False
s = ""
comps = name.rstrip("/").split("/")[:-1]
if comps[0] == "":
s = "/"
for c in comps:
if s and s[-1] != "/":
s += "/"
s += c
try:
os.mkdir(s)
ret = True
except OSError as e:
if e.args[0] != errno.EEXIST and e.args[0] != errno.EISDIR:
raise
ret = False
return ret
def save_file(fname, subf):
global file_buf
with open(fname, "wb") as outf:
while True:
sz = subf.readinto(file_buf)
if not sz:
break
outf.write(file_buf, sz)
def install_tar(f, prefix):
meta = {}
for info in f:
#print(info)
fname = info.name
try:
fname = fname[fname.index("/") + 1:]
except ValueError:
fname = ""
save = True
for p in ("setup.", "PKG-INFO", "README"):
#print(fname, p)
if fname.startswith(p) or ".egg-info" in fname:
if fname.endswith("/requires.txt"):
meta["deps"] = f.extractfile(info).read()
save = False
if debug:
print("Skipping", fname)
break
if save:
outfname = prefix + fname
if info.type != tarfile.DIRTYPE:
if debug:
print("Extracting " + outfname)
_makedirs(outfname)
subf = f.extractfile(info)
save_file(outfname, subf)
return meta
def expandhome(s):
if "~/" in s:
h = os.getenv("HOME")
s = s.replace("~/", h + "/")
return s
import ussl
import usocket
warn_ussl = True
def url_open(url):
global warn_ussl
proto, _, host, urlpath = url.split('/', 3)
ai = usocket.getaddrinfo(host, 443)
#print("Address infos:", ai)
addr = ai[0][4]
s = usocket.socket(ai[0][0])
#print("Connect address:", addr)
s.connect(addr)
if proto == "https:":
s = ussl.wrap_socket(s)
if warn_ussl:
print("Warning: %s SSL certificate is not validated" % host)
warn_ussl = False
# MicroPython rawsocket module supports file interface directly
s.write("GET /%s HTTP/1.0\r\nHost: %s\r\n\r\n" % (urlpath, host))
l = s.readline()
protover, status, msg = l.split(None, 2)
if status != b"200":
if status == b"404":
print("Package not found")
raise ValueError(status)
while 1:
l = s.readline()
if not l:
raise ValueError("Unexpected EOF")
if l == b'\r\n':
break
return s
def get_pkg_metadata(name):
f = url_open("https://pypi.python.org/pypi/%s/json" % name)
s = f.read()
f.close()
return json.loads(s)
def fatal(msg):
print(msg)
sys.exit(1)
def install_pkg(pkg_spec, install_path):
data = get_pkg_metadata(pkg_spec)
latest_ver = data["info"]["version"]
packages = data["releases"][latest_ver]
del data
gc.collect()
assert len(packages) == 1
package_url = packages[0]["url"]
print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
package_fname = op_basename(package_url)
f1 = url_open(package_url)
f2 = uzlib.DecompIO(f1, gzdict_sz)
f3 = tarfile.TarFile(fileobj=f2)
meta = install_tar(f3, install_path)
f1.close()
del f3
del f2
gc.collect()
return meta
def install(to_install, install_path=None):
# Calculate gzip dictionary size to use
global gzdict_sz
sz = gc.mem_free() + gc.mem_alloc()
if sz <= 65536:
gzdict_sz = 16 + 12
if install_path is None:
install_path = get_install_path()
if install_path[-1] != "/":
install_path += "/"
if not isinstance(to_install, list):
to_install = [to_install]
print("Installing to: " + install_path)
# sets would be perfect here, but don't depend on them
installed = []
try:
while to_install:
if debug:
print("Queue:", to_install)
pkg_spec = to_install.pop(0)
if pkg_spec in installed:
continue
meta = install_pkg(pkg_spec, install_path)
installed.append(pkg_spec)
if debug:
print(meta)
deps = meta.get("deps", "").rstrip()
if deps:
deps = deps.decode("utf-8").split("\n")
to_install.extend(deps)
except NotFoundError:
print("Error: cannot find '%s' package (or server error), packages may be partially installed" \
% pkg_spec, file=sys.stderr)
def get_install_path():
global install_path
if install_path is None:
# sys.path[0] is current module's path
install_path = sys.path[1]
install_path = expandhome(install_path)
return install_path
def cleanup():
for fname in cleanup_files:
try:
os.unlink(fname)
except OSError:
print("Warning: Cannot delete " + fname)
def help():
print("""\
upip - Simple PyPI package manager for MicroPython
Usage: micropython -m upip install [-p <path>] <package>... | -r <requirements.txt>
import upip; upip.install(package_or_list, [<path>])
If <path> is not given, packages will be installed into sys.path[1]
(can be set from MICROPYPATH environment variable, if current system
supports that).""")
print("Current value of sys.path[1]:", sys.path[1])
print("""\
Note: only MicroPython packages (usually, named micropython-*) are supported
for installation, upip does not support arbitrary code in setup.py.
""")
def main():
global debug
global install_path
install_path = None
if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help":
help()
return
if sys.argv[1] != "install":
fatal("Only 'install' command supported")
to_install = []
i = 2
while i < len(sys.argv) and sys.argv[i][0] == "-":
opt = sys.argv[i]
i += 1
if opt == "-h" or opt == "--help":
help()
return
elif opt == "-p":
install_path = sys.argv[i]
i += 1
elif opt == "-r":
list_file = sys.argv[i]
i += 1
with open(list_file) as f:
while True:
l = f.readline()
if not l:
break
to_install.append(l.rstrip())
elif opt == "--debug":
debug = True
else:
fatal("Unknown/unsupported option: " + opt)
to_install.extend(sys.argv[i:])
if not to_install:
help()
return
install(to_install)
if not debug:
cleanup()
if __name__ == "__main__":
main()
import uctypes
# http://www.gnu.org/software/tar/manual/html_node/Standard.html
TAR_HEADER = {
"name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100),
"size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12),
}
DIRTYPE = "dir"
REGTYPE = "file"
def roundup(val, align):
return (val + align - 1) & ~(align - 1)
class FileSection:
def __init__(self, f, content_len, aligned_len):
self.f = f
self.content_len = content_len
self.align = aligned_len - content_len
def read(self, sz=65536):
if self.content_len == 0:
return b""
if sz > self.content_len:
sz = self.content_len
data = self.f.read(sz)
sz = len(data)
self.content_len -= sz
return data
def readinto(self, buf):
if self.content_len == 0:
return 0
if len(buf) > self.content_len:
buf = memoryview(buf)[:self.content_len]
sz = self.f.readinto(buf)
self.content_len -= sz
return sz
def skip(self):
sz = self.content_len + self.align
if sz:
buf = bytearray(16)
while sz:
s = min(sz, 16)
self.f.readinto(buf, s)
sz -= s
class TarInfo:
def __str__(self):
return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size)
class TarFile:
def __init__(self, name=None, fileobj=None):
if fileobj:
self.f = fileobj
else:
self.f = open(name, "rb")
self.subf = None
def next(self):
if self.subf:
self.subf.skip()
buf = self.f.read(512)
if not buf:
return None
h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN)
# Empty block means end of archive
if h.name[0] == 0:
return None
d = TarInfo()
d.name = str(h.name, "utf-8").rstrip()
d.size = int(bytes(h.size).rstrip(), 8)
d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"]
self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512))
return d
def __iter__(self):
return self
def __next__(self):
v = self.next()
if v is None:
raise StopIteration
return v
def extractfile(self, tarinfo):
return tarinfo.subf
-include mpconfigport.mk
include ../py/mkenv.mk
FROZEN_DIR = scripts
# define main target
PROG = micropython
......@@ -148,17 +150,6 @@ SRC_C = \
fatfs_port.c \
$(SRC_MOD)
# Include builtin package manager in the standard build (and coverage)
ifeq ($(PROG),micropython)
SRC_C += $(BUILD)/_frozen_upip.c
else ifeq ($(PROG),micropython_coverage)
SRC_C += $(BUILD)/_frozen_upip.c
else ifeq ($(PROG), micropython_nanbox)
SRC_C += $(BUILD)/_frozen_upip.c
else ifeq ($(PROG), micropython_freedos)
SRC_C += $(BUILD)/_frozen_upip.c
endif
LIB_SRC_C = $(addprefix lib/,\
$(LIB_SRC_C_EXTRA) \
timeutils/timeutils.c \
......@@ -235,7 +226,7 @@ fast:
# build a minimal interpreter
minimal:
$(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE="<mpconfigport_minimal.h>"' \
BUILD=build-minimal PROG=micropython_minimal \
BUILD=build-minimal PROG=micropython_minimal FROZEN_DIR= \
MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \
MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \
MICROPY_USE_READLINE=0 MICROPY_FATFS=0
......@@ -272,22 +263,6 @@ coverage_test: coverage
gcov -o build-coverage/py ../py/*.c
gcov -o build-coverage/extmod ../extmod/*.c
$(BUILD)/_frozen_upip.c: $(BUILD)/frozen_upip/upip.py
$(MAKE_FROZEN) $(dir $^) > $@
# Select latest upip version available
UPIP_TARBALL := $(shell ls -1 -v ../tools/micropython-upip-*.tar.gz | tail -n1)
$(BUILD)/frozen_upip/upip.py: $(UPIP_TARBALL)
$(ECHO) "MISC Preparing upip as frozen module"
$(Q)mkdir -p $(BUILD)
$(Q)rm -rf $(BUILD)/micropython-upip-*
$(Q)tar -C $(BUILD) -xz -f $^
$(Q)rm -rf $(dir $@)
$(Q)mkdir -p $(dir $@)
$(Q)cp $(BUILD)/micropython-upip-*/upip*.py $(dir $@)
# Value of configure's --host= option (required for cross-compilation).
# Deduce it from CROSS_COMPILE by default, but can be overriden.
ifneq ($(CROSS_COMPILE),)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment