From d053ef783159e6d862c101f78f420896ed2bdb37 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Sun, 20 Apr 2014 16:45:40 +0200 Subject: [PATCH] move scripts into buildozer.scripts.*, and use console_scripts for setup() --- buildozer/__init__.py | 279 +-------------------------------- buildozer/jsonstore.py | 49 ++++++ buildozer/scripts/__init__.py | 0 buildozer/scripts/client.py | 22 +++ buildozer/scripts/remote.py | 286 ++++++++++++++++++++++++++++++++++ setup.py | 50 +++++- tools/buildozer | 5 - tools/buildozer-remote | 5 - 8 files changed, 404 insertions(+), 292 deletions(-) create mode 100644 buildozer/jsonstore.py create mode 100644 buildozer/scripts/__init__.py create mode 100644 buildozer/scripts/client.py create mode 100644 buildozer/scripts/remote.py delete mode 100755 tools/buildozer delete mode 100755 tools/buildozer-remote diff --git a/buildozer/__init__.py b/buildozer/__init__.py index 9f0324f..e14b697 100644 --- a/buildozer/__init__.py +++ b/buildozer/__init__.py @@ -1,12 +1,8 @@ ''' +Buildozer +========= -Layout directory for buildozer: - - build/ - / - platform/ - all the platform files necessary - app/ - compiled application - +Generic Python packager for Android / iOS. Desktop later. ''' @@ -15,13 +11,11 @@ __version__ = '0.11-dev' import fcntl import os import re -import socket import sys import zipfile -import io from select import select from buildozer.jsonstore import JsonStore -from sys import stdout, stderr, stdin, exit +from sys import stdout, stderr, exit from re import search from os.path import join, exists, dirname, realpath, splitext, expanduser from subprocess import Popen, PIPE @@ -36,14 +30,6 @@ except ImportError: from urllib import FancyURLopener from ConfigParser import SafeConfigParser -# windows does not have termios... -try: - import termios - import tty - has_termios = True -except ImportError: - has_termios = False - RESET_SEQ = "\033[0m" COLOR_SEQ = "\033[1;{0}m" BOLD_SEQ = "\033[1m" @@ -1062,263 +1048,6 @@ class Buildozer(object): return default return self.config.getboolean(section, token) - -class BuildozerRemote(Buildozer): - def run_command(self, args): - while args: - if not args[0].startswith('-'): - break - arg = args.pop(0) - - if arg in ('-v', '--verbose'): - self.log_level = 2 - - elif arg in ('-p', '--profile'): - self.config_profile = args.pop(0) - - elif arg in ('-h', '--help'): - self.usage() - exit(0) - - elif arg == '--version': - print('Buildozer (remote) {0}'.format(__version__)) - exit(0) - - self._merge_config_profile() - - if len(args) < 2: - self.usage() - return - - remote_name = args[0] - remote_section = 'remote:{}'.format(remote_name) - if not self.config.has_section(remote_section): - self.error('Unknown remote "{}", must be configured first.'.format( - remote_name)) - return - - self.remote_host = remote_host = self.config.get( - remote_section, 'host', '') - self.remote_port = self.config.get( - remote_section, 'port', '22') - self.remote_user = remote_user = self.config.get( - remote_section, 'user', '') - self.remote_build_dir = remote_build_dir = self.config.get( - remote_section, 'build_directory', '') - self.remote_identity = self.config.get( - remote_section, 'identity', '') - if not remote_host: - self.error('Missing "host = " for {}'.format(remote_section)) - return - if not remote_user: - self.error('Missing "user = " for {}'.format(remote_section)) - return - if not remote_build_dir: - self.error('Missing "build_directory = " for {}'.format(remote_section)) - return - - # fake the target - self.targetname = 'remote' - self.check_build_layout() - - # prepare our source code - self.info('Prepare source code to sync') - self._copy_application_sources() - self._ssh_connect() - try: - self._ensure_buildozer() - self._sync_application_sources() - self._do_remote_commands(args[1:]) - self._ssh_sync(os.getcwd(), mode='get') - finally: - self._ssh_close() - - def _ssh_connect(self): - self.info('Connecting to {}'.format(self.remote_host)) - import paramiko - self._ssh_client = client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.load_system_host_keys() - kwargs = {} - if self.remote_identity: - kwargs['key_filename'] = expanduser(self.remote_identity) - client.connect(self.remote_host, username=self.remote_user, - port=int(self.remote_port), **kwargs) - self._sftp_client = client.open_sftp() - - def _ssh_close(self): - self.debug('Closing remote connection') - self._sftp_client.close() - self._ssh_client.close() - - def _ensure_buildozer(self): - s = self._sftp_client - root_dir = s.normalize('.') - self.remote_build_dir = join(root_dir, self.remote_build_dir, - self.package_full_name) - self.debug('Remote build directory: {}'.format(self.remote_build_dir)) - self._ssh_mkdir(self.remote_build_dir) - self._ssh_sync(__path__[0]) - - def _sync_application_sources(self): - self.info('Synchronize application sources') - self._ssh_sync(self.app_dir) - - # create custom buildozer.spec - self.info('Create custom buildozer.spec') - config = SafeConfigParser() - config.read('buildozer.spec') - config.set('app', 'source.dir', 'app') - - fn = join(self.remote_build_dir, 'buildozer.spec') - fd = self._sftp_client.open(fn, 'wb') - config.write(fd) - fd.close() - - def _do_remote_commands(self, args): - self.info('Execute remote buildozer') - cmd = ( - 'source ~/.profile;' - 'cd {0};' - 'env PYTHONPATH={0}:$PYTHONPATH ' - 'python -c "import buildozer, sys;' - 'buildozer.Buildozer().run_command(sys.argv[1:])" {1} {2} 2>&1').format( - self.remote_build_dir, - '--verbose' if self.log_level == 2 else '', - ' '.join(args), - ) - self._ssh_command(cmd) - - def _ssh_mkdir(self, *args): - directory = join(*args) - self.debug('Create remote directory {}'.format(directory)) - try: - self._sftp_client.mkdir(directory) - except IOError: - # already created? - try: - self._sftp_client.stat(directory) - except IOError: - self.error('Unable to create remote directory {}'.format(directory)) - raise - - def _ssh_sync(self, directory, mode='put'): - self.debug('Syncing {} directory'.format(directory)) - directory = realpath(directory) - base_strip = directory.rfind('/') - if mode == 'get': - local_dir = join(directory,'bin') - remote_dir = join(self.remote_build_dir, 'bin') - if not os.path.exists(local_dir): - os.path.makedirs(local_dir) - for _file in self._sftp_client.listdir(path=remote_dir): - self._sftp_client.get(join(remote_dir, _file), - join(local_dir, _file)) - return - for root, dirs, files in walk(directory): - self._ssh_mkdir(self.remote_build_dir, root[base_strip + 1:]) - for fn in files: - if splitext(fn)[1] in ('.pyo', '.pyc', '.swp'): - continue - local_file = join(root, fn) - remote_file = join(self.remote_build_dir, root[base_strip + 1:], fn) - self.debug('Sync {} -> {}'.format(local_file, remote_file)) - self._sftp_client.put(local_file, remote_file) - - def _ssh_command(self, command): - self.debug('Execute remote command {}'.format(command)) - #shell = self._ssh_client.invoke_shell() - #shell.sendall(command) - #shell.sendall('\nexit\n') - transport = self._ssh_client.get_transport() - channel = transport.open_session() - try: - channel.exec_command(command) - self._interactive_shell(channel) - finally: - channel.close() - - def _interactive_shell(self, chan): - if has_termios: - self._posix_shell(chan) - else: - self._windows_shell(chan) - - def _posix_shell(self, chan): - oldtty = termios.tcgetattr(stdin) - try: - #tty.setraw(stdin.fileno()) - #tty.setcbreak(stdin.fileno()) - chan.settimeout(0.0) - - while True: - r, w, e = select([chan, stdin], [], []) - if chan in r: - try: - x = chan.recv(128) - if len(x) == 0: - print('\r\n*** EOF\r\n',) - break - stdout.write(x) - stdout.flush() - #print len(x), repr(x) - except socket.timeout: - pass - if stdin in r: - x = stdin.read(1) - if len(x) == 0: - break - chan.sendall(x) - finally: - termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) - - # thanks to Mike Looijmans for this code - def _windows_shell(self,chan): - import threading - - stdout.write("Line-buffered terminal emulation. Press F6 or ^Z to send EOF.\r\n\r\n") - - def writeall(sock): - while True: - data = sock.recv(256) - if not data: - stdout.write('\r\n*** EOF ***\r\n\r\n') - stdout.flush() - break - stdout.write(data) - stdout.flush() - - writer = threading.Thread(target=writeall, args=(chan,)) - writer.start() - - try: - while True: - d = stdin.read(1) - if not d: - break - chan.send(d) - except EOFError: - # user hit ^Z or F6 - pass - -def run(): - try: - Buildozer().run_command(sys.argv[1:]) - except BuildozerCommandException: - # don't show the exception in the command line. The log already show the - # command failed. - pass - except BuildozerException as error: - Buildozer().error('%s' % error) - -def run_remote(): - try: - BuildozerRemote().run_command(sys.argv[1:]) - except BuildozerCommandException: - pass - except BuildozerException as error: - Buildozer().error('%s' % error) - def set_config_from_envs(config): '''Takes a ConfigParser, and checks every section/token for an environment variable of the form SECTION_TOKEN, with any dots diff --git a/buildozer/jsonstore.py b/buildozer/jsonstore.py new file mode 100644 index 0000000..0be90ce --- /dev/null +++ b/buildozer/jsonstore.py @@ -0,0 +1,49 @@ +""" +Replacement for shelve, using json. +This is currently needed to correctly support db between Python 2 and 3. +""" + +__all__ = ["JsonStore"] + +import io +from json import load, dump +from os.path import exists + + +class JsonStore(object): + + def __init__(self, filename): + super(JsonStore, self).__init__() + self.filename = filename + self.data = {} + if exists(filename): + try: + with io.open(filename, encoding='utf-8') as fd: + self.data = load(fd) + except ValueError: + print("Unable to read the state.db, content will be replaced.") + + def __getitem__(self, key): + return self.data[key] + + def __setitem__(self, key, value): + self.data[key] = value + self.sync() + + def __delitem__(self, key): + del self.data[key] + self.sync() + + def __contains__(self, item): + return item in self.data + + def get(self, item, default=None): + return self.data.get(item, default) + + def keys(self): + return self.data.keys() + + def sync(self): + with open(self.filename, 'w', encoding='utf-8') as fd: + dump(self.data, fd) + diff --git a/buildozer/scripts/__init__.py b/buildozer/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildozer/scripts/client.py b/buildozer/scripts/client.py new file mode 100644 index 0000000..e30a9b4 --- /dev/null +++ b/buildozer/scripts/client.py @@ -0,0 +1,22 @@ +''' +Main Buildozer client +===================== + +''' + +import sys +from buildozer import Buildozer, BuildozerCommandException, BuildozerException + + +def main(): + try: + Buildozer().run_command(sys.argv[1:]) + except BuildozerCommandException: + # don't show the exception in the command line. The log already show + # the command failed. + pass + except BuildozerException as error: + Buildozer().error('%s' % error) + +if __name__ == '__main__': + main() diff --git a/buildozer/scripts/remote.py b/buildozer/scripts/remote.py new file mode 100644 index 0000000..44fefb0 --- /dev/null +++ b/buildozer/scripts/remote.py @@ -0,0 +1,286 @@ +''' +Buildozer remote +================ + +.. warning:: + + This is an experimental tool and not widely used. It might not fit for you. + +Pack and send the source code to a remote SSH server, bundle buildozer with it, +and start the build on the remote. +You need paramiko to make it work. +''' + +__all__ = ["BuildozerRemote"] + +import socket +import sys +from buildozer import ( + Buildozer, BuildozerCommandException, BuildozerException, __version__) +from sys import stdout, stdin, exit +from select import select +from os.path import join, expanduser, realpath, exists, splitext +from os import makedirs, walk, getcwd +try: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import SafeConfigParser +try: + import termios + import tty + has_termios = True +except ImportError: + has_termios = False +try: + import paramiko +except ImportError: + print('Paramiko missing: pip install paramiko') + + +class BuildozerRemote(Buildozer): + def run_command(self, args): + while args: + if not args[0].startswith('-'): + break + arg = args.pop(0) + + if arg in ('-v', '--verbose'): + self.log_level = 2 + + elif arg in ('-p', '--profile'): + self.config_profile = args.pop(0) + + elif arg in ('-h', '--help'): + self.usage() + exit(0) + + elif arg == '--version': + print('Buildozer (remote) {0}'.format(__version__)) + exit(0) + + self._merge_config_profile() + + if len(args) < 2: + self.usage() + return + + remote_name = args[0] + remote_section = 'remote:{}'.format(remote_name) + if not self.config.has_section(remote_section): + self.error('Unknown remote "{}", must be configured first.'.format( + remote_name)) + return + + self.remote_host = remote_host = self.config.get( + remote_section, 'host', '') + self.remote_port = self.config.get( + remote_section, 'port', '22') + self.remote_user = remote_user = self.config.get( + remote_section, 'user', '') + self.remote_build_dir = remote_build_dir = self.config.get( + remote_section, 'build_directory', '') + self.remote_identity = self.config.get( + remote_section, 'identity', '') + if not remote_host: + self.error('Missing "host = " for {}'.format(remote_section)) + return + if not remote_user: + self.error('Missing "user = " for {}'.format(remote_section)) + return + if not remote_build_dir: + self.error('Missing "build_directory = " for {}'.format(remote_section)) + return + + # fake the target + self.targetname = 'remote' + self.check_build_layout() + + # prepare our source code + self.info('Prepare source code to sync') + self._copy_application_sources() + self._ssh_connect() + try: + self._ensure_buildozer() + self._sync_application_sources() + self._do_remote_commands(args[1:]) + self._ssh_sync(getcwd(), mode='get') + finally: + self._ssh_close() + + def _ssh_connect(self): + self.info('Connecting to {}'.format(self.remote_host)) + self._ssh_client = client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.load_system_host_keys() + kwargs = {} + if self.remote_identity: + kwargs['key_filename'] = expanduser(self.remote_identity) + client.connect(self.remote_host, username=self.remote_user, + port=int(self.remote_port), **kwargs) + self._sftp_client = client.open_sftp() + + def _ssh_close(self): + self.debug('Closing remote connection') + self._sftp_client.close() + self._ssh_client.close() + + def _ensure_buildozer(self): + s = self._sftp_client + root_dir = s.normalize('.') + self.remote_build_dir = join(root_dir, self.remote_build_dir, + self.package_full_name) + self.debug('Remote build directory: {}'.format(self.remote_build_dir)) + self._ssh_mkdir(self.remote_build_dir) + self._ssh_sync(__path__[0]) + + def _sync_application_sources(self): + self.info('Synchronize application sources') + self._ssh_sync(self.app_dir) + + # create custom buildozer.spec + self.info('Create custom buildozer.spec') + config = SafeConfigParser() + config.read('buildozer.spec') + config.set('app', 'source.dir', 'app') + + fn = join(self.remote_build_dir, 'buildozer.spec') + fd = self._sftp_client.open(fn, 'wb') + config.write(fd) + fd.close() + + def _do_remote_commands(self, args): + self.info('Execute remote buildozer') + cmd = ( + 'source ~/.profile;' + 'cd {0};' + 'env PYTHONPATH={0}:$PYTHONPATH ' + 'python -c "import buildozer, sys;' + 'buildozer.Buildozer().run_command(sys.argv[1:])" {1} {2} 2>&1').format( + self.remote_build_dir, + '--verbose' if self.log_level == 2 else '', + ' '.join(args), + ) + self._ssh_command(cmd) + + def _ssh_mkdir(self, *args): + directory = join(*args) + self.debug('Create remote directory {}'.format(directory)) + try: + self._sftp_client.mkdir(directory) + except IOError: + # already created? + try: + self._sftp_client.stat(directory) + except IOError: + self.error('Unable to create remote directory {}'.format(directory)) + raise + + def _ssh_sync(self, directory, mode='put'): + self.debug('Syncing {} directory'.format(directory)) + directory = realpath(directory) + base_strip = directory.rfind('/') + if mode == 'get': + local_dir = join(directory,'bin') + remote_dir = join(self.remote_build_dir, 'bin') + if not exists(local_dir): + makedirs(local_dir) + for _file in self._sftp_client.listdir(path=remote_dir): + self._sftp_client.get(join(remote_dir, _file), + join(local_dir, _file)) + return + for root, dirs, files in walk(directory): + self._ssh_mkdir(self.remote_build_dir, root[base_strip + 1:]) + for fn in files: + if splitext(fn)[1] in ('.pyo', '.pyc', '.swp'): + continue + local_file = join(root, fn) + remote_file = join(self.remote_build_dir, root[base_strip + 1:], fn) + self.debug('Sync {} -> {}'.format(local_file, remote_file)) + self._sftp_client.put(local_file, remote_file) + + def _ssh_command(self, command): + self.debug('Execute remote command {}'.format(command)) + #shell = self._ssh_client.invoke_shell() + #shell.sendall(command) + #shell.sendall('\nexit\n') + transport = self._ssh_client.get_transport() + channel = transport.open_session() + try: + channel.exec_command(command) + self._interactive_shell(channel) + finally: + channel.close() + + def _interactive_shell(self, chan): + if has_termios: + self._posix_shell(chan) + else: + self._windows_shell(chan) + + def _posix_shell(self, chan): + oldtty = termios.tcgetattr(stdin) + try: + #tty.setraw(stdin.fileno()) + #tty.setcbreak(stdin.fileno()) + chan.settimeout(0.0) + + while True: + r, w, e = select([chan, stdin], [], []) + if chan in r: + try: + x = chan.recv(128) + if len(x) == 0: + print('\r\n*** EOF\r\n',) + break + stdout.write(x) + stdout.flush() + #print len(x), repr(x) + except socket.timeout: + pass + if stdin in r: + x = stdin.read(1) + if len(x) == 0: + break + chan.sendall(x) + finally: + termios.tcsetattr(stdin, termios.TCSADRAIN, oldtty) + + # thanks to Mike Looijmans for this code + def _windows_shell(self,chan): + import threading + + stdout.write("Line-buffered terminal emulation. Press F6 or ^Z to send EOF.\r\n\r\n") + + def writeall(sock): + while True: + data = sock.recv(256) + if not data: + stdout.write('\r\n*** EOF ***\r\n\r\n') + stdout.flush() + break + stdout.write(data) + stdout.flush() + + writer = threading.Thread(target=writeall, args=(chan,)) + writer.start() + + try: + while True: + d = stdin.read(1) + if not d: + break + chan.send(d) + except EOFError: + # user hit ^Z or F6 + pass + +def main(): + try: + BuildozerRemote().run_command(sys.argv[1:]) + except BuildozerCommandException: + pass + except BuildozerException as error: + Buildozer().error('%s' % error) + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 3cf70a2..5b75e4b 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,33 @@ -from distutils.core import setup +''' +Buildozer +''' -import buildozer +from setuptools import setup +import codecs +import os +import re + +here = os.path.abspath(os.path.dirname(__file__)) + + +def find_version(*file_paths): + # Open in Latin-1 so that we avoid encoding errors. + # Use codecs.open for Python 2 compatibility + with codecs.open(os.path.join(here, *file_paths), 'r', 'utf-8') as f: + version_file = f.read() + + # The version line must have the form + # __version__ = 'ver' + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") setup( name='buildozer', - version=buildozer.__version__, + version=find_version('buildozer', '__init__.py'), + description='Generic Python packager for Android / iOS and Desktop', author='Mathieu Virbel', author_email='mat@kivy.org', url='http://github.com/kivy/buildozer', @@ -12,8 +35,21 @@ setup( packages=[ 'buildozer', 'buildozer.targets', - 'buildozer.libs'], + 'buildozer.libs', + 'buildozer.scripts'], package_data={'buildozer': ['default.spec']}, - scripts=['tools/buildozer', 'tools/buildozer-remote'], - description='Generic Python packager for Android / iOS and Desktop' -) + include_package_data=True, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.1', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3'], + entry_points={ + 'console_scripts': [ + 'buildozer=buildozer.scripts.client:main', + 'buildozer-remote=buildozer.scripts.remote:main']}) diff --git a/tools/buildozer b/tools/buildozer deleted file mode 100755 index 678521e..0000000 --- a/tools/buildozer +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python2.7 - -if __name__ == '__main__': - from buildozer import run - run() diff --git a/tools/buildozer-remote b/tools/buildozer-remote deleted file mode 100755 index 2990763..0000000 --- a/tools/buildozer-remote +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python2.7 - -if __name__ == '__main__': - from buildozer import run_remote - run_remote()