--- README.md.orig 2014-11-19 06:36:44 UTC +++ README.md @@ -5,7 +5,7 @@ The bench allows you to setup Frappe / E To do this install, you must have basic information on how Linux works and should be able to use the command-line. If you are looking easier ways to get started and evaluate ERPNext, [download the Virtual Machine or take a free trial at FrappeCloud.com](https://erpnext.com/use). -For questions, please join the [developer forum](https://groups.google.com/group/erpnext-developer-forum). +For questions, please join the [developer forum](https://discuss.frappe.io/). Installation ============ @@ -37,6 +37,15 @@ Install pre-requisites, * Redis * [wkhtmltopdf](http://wkhtmltopdf.org/downloads.html) (optional, required for pdf generation) * Memcached + +For installing MaraiDB on OSX, use: +``` +brew install mariadb +mysql_install_db +mysql.server start +mysqladmin -uroot password ROOTPASSWORD +``` + Install bench as a *non root* user, @@ -105,11 +114,9 @@ To setup a bench that runs ERPNext, run cd ~ bench init frappe-bench cd frappe-bench -bench get-app erpnext https://github.com/frappe/erpnext # Add ERPNext to your bench apps -bench get-app shopping_cart https://github.com/frappe/shopping-cart # Add Shopping cart to your bench apps -bench new-site site1.local # Create a new site -bench frappe --install_app erpnext site1.local # Install ERPNext for the site -bench frappe --install_app shopping_cart site1.local # Install Shopping cart for the site +bench get-app erpnext https://github.com/frappe/erpnext # Add ERPNext to your bench apps +bench new-site site1.local # Create a new site +bench install-app erpnext # Install ERPNext for the site ``` You can now either use `bench start` or setup the bench for production use. @@ -162,7 +169,7 @@ Frappe Processes * WSGI Server * The WSGI server is responsible for responding to the HTTP requests to - frappe. In development scenario (`frappe --serve` or `bench start`), the + frappe. In development scenario (`bench serve` or `bench start`), the Werkzeug WSGI server is used and in production, gunicorn (automatically configured in supervisor) is used. --- bench/app.py.orig 2014-11-19 06:36:44 UTC +++ bench/app.py @@ -1,12 +1,22 @@ import os -from .utils import exec_cmd, get_frappe, check_git_for_shallow_clone, get_config, build_assets, restart_supervisor_processes, get_cmd_output +from .utils import exec_cmd, get_frappe, check_git_for_shallow_clone, get_config, build_assets, restart_supervisor_processes, get_cmd_output, run_frappe_cmd import logging import requests +import semantic_version import json +import re +import subprocess + logger = logging.getLogger(__name__) +class MajorVersionUpgradeException(Exception): + def __init__(self, message, upstream_version, local_version): + super(MajorVersionUpgradeException, self).__init__(message) + self.upstream_version = upstream_version + self.local_version = local_version + def get_apps(bench='.'): try: with open(os.path.join(bench, 'sites', 'apps.txt')) as f: @@ -18,10 +28,19 @@ def add_to_appstxt(app, bench='.'): apps = get_apps(bench=bench) if app not in apps: apps.append(app) - with open(os.path.join(bench, 'sites', 'apps.txt'), 'w') as f: - return f.write('\n'.join(apps)) + return write_appstxt(apps, bench=bench) -def get_app(app, git_url, branch=None, bench='.'): +def remove_from_appstxt(app, bench='.'): + apps = get_apps(bench=bench) + if app in apps: + apps.remove(app) + return write_appstxt(apps, bench=bench) + +def write_appstxt(apps, bench='.'): + with open(os.path.join(bench, 'sites', 'apps.txt'), 'w') as f: + return f.write('\n'.join(apps)) + +def get_app(app, git_url, branch=None, bench='.', build_asset_files=True): logger.info('getting app {}'.format(app)) shallow_clone = '--depth 1' if check_git_for_shallow_clone() and get_config().get('shallow_clone') else '' branch = '--branch {branch}'.format(branch=branch) if branch else '' @@ -33,14 +52,20 @@ def get_app(app, git_url, branch=None, b cwd=os.path.join(bench, 'apps')) print 'installing', app install_app(app, bench=bench) - build_assets(bench=bench) + if build_asset_files: + build_assets(bench=bench) conf = get_config() if conf.get('restart_supervisor_on_update'): restart_supervisor_processes(bench=bench) def new_app(app, bench='.'): logger.info('creating new app {}'.format(app)) - exec_cmd("{frappe} --make_app {apps}".format(frappe=get_frappe(bench=bench), apps=os.path.join(bench, 'apps'))) + apps = os.path.abspath(os.path.join(bench, 'apps')) + if FRAPPE_VERSION == 4: + exec_cmd("{frappe} --make_app {apps} {app}".format(frappe=get_frappe(bench=bench), + apps=apps, app=app)) + else: + run_frappe_cmd('make-app', apps, app, bench=bench) install_app(app, bench=bench) def install_app(app, bench='.'): @@ -57,19 +82,112 @@ def pull_all_apps(bench='.'): apps_dir = os.path.join(bench, 'apps') apps = [app for app in os.listdir(apps_dir) if os.path.isdir(os.path.join(apps_dir, app))] rebase = '--rebase' if get_config().get('rebase_on_pull') else '' + frappe_dir = os.path.join(apps_dir, 'frappe') + for app in apps: app_dir = os.path.join(apps_dir, app) if os.path.exists(os.path.join(app_dir, '.git')): logger.info('pulling {0}'.format(app)) exec_cmd("git pull {rebase} upstream {branch}".format(rebase=rebase, branch=get_current_branch(app_dir)), cwd=app_dir) +def is_version_upgrade(bench='.', branch=None): + apps_dir = os.path.join(bench, 'apps') + frappe_dir = os.path.join(apps_dir, 'frappe') + + fetch_upstream(frappe_dir) + upstream_version = get_upstream_version(frappe_dir, branch=branch) + + if not upstream_version: + raise Exception("Current branch of 'frappe' not in upstream") + + local_version = get_major_version(get_current_version(frappe_dir)) + upstream_version = get_major_version(upstream_version) + + if upstream_version - local_version > 0: + return (local_version, upstream_version) + return False + +def get_current_frappe_version(bench='.'): + apps_dir = os.path.join(bench, 'apps') + frappe_dir = os.path.join(apps_dir, 'frappe') + + try: + return get_major_version(get_current_version(frappe_dir)) + except IOError: + return '' + def get_current_branch(repo_dir): return get_cmd_output("basename $(git symbolic-ref -q HEAD)", cwd=repo_dir) +def fetch_upstream(repo_dir): + return exec_cmd("git fetch upstream", cwd=repo_dir) + +def get_current_version(repo_dir): + with open(os.path.join(repo_dir, 'setup.py')) as f: + return get_version_from_string(f.read()) + +def get_upstream_version(repo_dir, branch=None): + if not branch: + branch = get_current_branch(repo_dir) + try: + contents = subprocess.check_output(['git', 'show', 'upstream/{branch}:setup.py'.format(branch=branch)], cwd=repo_dir, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, e: + if "Invalid object" in e.output: + return None + else: + raise + return get_version_from_string(contents) + +def switch_branch(branch, apps=None, bench='.', upgrade=False): + from .utils import update_requirements, backup_all_sites, patch_sites, build_assets, pre_upgrade, post_upgrade + import utils + apps_dir = os.path.join(bench, 'apps') + version_upgrade = is_version_upgrade(bench=bench, branch=branch) + if version_upgrade and not upgrade: + raise MajorVersionUpgradeException("Switching to {0} will cause upgrade from {1} to {2}. Pass --upgrade to confirm".format(branch, version_upgrade[0], version_upgrade[1]), version_upgrade[0], version_upgrade[1]) + + if not apps: + apps = ('frappe', 'erpnext', 'shopping_cart') + for app in apps: + app_dir = os.path.join(apps_dir, app) + if os.path.exists(app_dir): + unshallow = "--unshallow" if os.path.exists(os.path.join(app_dir, ".git", "shallow")) else "" + exec_cmd("git config --unset-all remote.upstream.fetch", cwd=app_dir) + exec_cmd("git config --add remote.upstream.fetch '+refs/heads/*:refs/remotes/upstream/*'", cwd=app_dir) + exec_cmd("git fetch upstream {unshallow}".format(unshallow=unshallow), cwd=app_dir) + exec_cmd("git checkout {branch}".format(branch=branch), cwd=app_dir) + exec_cmd("git merge upstream/{branch}".format(branch=branch), cwd=app_dir) + + if version_upgrade and upgrade: + update_requirements() + pre_upgrade(version_upgrade[0], version_upgrade[1]) + reload(utils) + backup_all_sites() + patch_sites() + build_assets() + post_upgrade(version_upgrade[0], version_upgrade[1]) + +def switch_to_master(apps=None, bench='.', upgrade=False): + switch_branch('master', apps=apps, bench=bench, upgrade=upgrade) + +def switch_to_develop(apps=None, bench='.', upgrade=False): + switch_branch('develop', apps=apps, bench=bench, upgrade=upgrade) + +def switch_to_v4(apps=None, bench='.', upgrade=False): + switch_branch('v4.x.x', apps=apps, bench=bench, upgrade=upgrade) + +def get_version_from_string(contents): + match = re.search(r"^(\s*%s\s*=\s*['\\\"])(.+?)(['\"])(?sm)" % 'version', + contents) + return match.group(2) + +def get_major_version(version): + return semantic_version.Version(version).major + def install_apps_from_path(path, bench='.'): apps = get_apps_json(path) for app in apps: - get_app(app['name'], app['url'], branch=app.get('branch'), bench=bench) + get_app(app['name'], app['url'], branch=app.get('branch'), bench=bench, build_asset_files=False) def get_apps_json(path): if path.startswith('http'): @@ -78,3 +196,5 @@ def get_apps_json(path): else: with open(path) as f: return json.load(f) + +FRAPPE_VERSION = get_current_frappe_version() --- bench/cli.py.orig 2014-11-19 06:36:44 UTC +++ bench/cli.py @@ -8,32 +8,54 @@ from .utils import setup_sudoers as _set from .utils import start as _start from .utils import setup_procfile as _setup_procfile from .utils import set_nginx_port as _set_nginx_port -from .utils import set_nginx_port as _set_nginx_port +from .utils import set_url_root as _set_url_root from .utils import set_default_site as _set_default_site -from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_frappe, setup_logging, +from .utils import (build_assets, patch_sites, exec_cmd, update_bench, get_env_cmd, get_frappe, setup_logging, get_config, update_config, restart_supervisor_processes, put_config, default_config, update_requirements, - backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges) + backup_all_sites, backup_site, get_sites, prime_wheel_cache, is_root, set_mariadb_host, drop_privileges, + fix_file_perms, fix_prod_setup_perms, set_ssl_certificate, set_ssl_certificate_key, get_cmd_output, post_upgrade, + pre_upgrade, PatchError, download_translations_p) from .app import get_app as _get_app from .app import new_app as _new_app -from .app import pull_all_apps -from .config import generate_nginx_config, generate_supervisor_config +from .app import pull_all_apps, get_apps, get_current_frappe_version, is_version_upgrade, switch_to_v4, switch_to_master, switch_to_develop +from .config import generate_nginx_config, generate_supervisor_config, generate_redis_config from .production_setup import setup_production as _setup_production +from .migrate_to_v5 import migrate_to_v5 import os import sys import logging import copy +import json import pwd import grp +import subprocess logger = logging.getLogger('bench') +global FRAPPE_VERSION + def cli(): check_uid() change_dir() change_uid() if len(sys.argv) > 2 and sys.argv[1] == "frappe": - return frappe() - return bench() + return old_frappe_cli() + elif len(sys.argv) > 1 and sys.argv[1] in get_frappe_commands(): + return frappe_cmd() + elif len(sys.argv) > 1 and sys.argv[1] in ("--site", "--verbose", "--force", "--profile"): + return frappe_cmd() + elif len(sys.argv) > 1 and sys.argv[1]=="--help": + print click.Context(bench).get_help() + print + print get_frappe_help() + return + elif len(sys.argv) > 1 and sys.argv[1] in get_apps(): + return app_cmd() + else: + try: + bench() + except PatchError: + sys.exit(1) def cmd_requires_root(): if len(sys.argv) > 2 and sys.argv[2] in ('production', 'sudoers'): @@ -57,17 +79,51 @@ def change_uid(): sys.exit(1) def change_dir(): + if os.path.exists('config.json') or "init" in sys.argv: + return dir_path_file = '/etc/frappe_bench_dir' if os.path.exists(dir_path_file): with open(dir_path_file) as f: dir_path = f.read().strip() - os.chdir(dir_path) + if os.path.exists(dir_path): + os.chdir(dir_path) -def frappe(bench='.'): +def old_frappe_cli(bench='.'): f = get_frappe(bench=bench) os.chdir(os.path.join(bench, 'sites')) os.execv(f, [f] + sys.argv[2:]) +def app_cmd(bench='.'): + f = get_env_cmd('python', bench=bench) + os.chdir(os.path.join(bench, 'sites')) + os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper'] + sys.argv[1:]) + +def frappe_cmd(bench='.'): + f = get_env_cmd('python', bench=bench) + os.chdir(os.path.join(bench, 'sites')) + os.execv(f, [f] + ['-m', 'frappe.utils.bench_helper', 'frappe'] + sys.argv[1:]) + +def get_frappe_commands(bench='.'): + python = get_env_cmd('python', bench=bench) + sites_path = os.path.join(bench, 'sites') + if not os.path.exists(sites_path): + return [] + try: + return json.loads(get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-commands".format(python=python), cwd=sites_path)) + except subprocess.CalledProcessError: + return [] + +def get_frappe_help(bench='.'): + python = get_env_cmd('python', bench=bench) + sites_path = os.path.join(bench, 'sites') + if not os.path.exists(sites_path): + return [] + try: + out = get_cmd_output("{python} -m frappe.utils.bench_helper get-frappe-help".format(python=python), cwd=sites_path) + return "Framework commands:\n" + out.split('Commands:')[1] + except subprocess.CalledProcessError: + return "" + @click.command() def shell(bench='.'): if not os.environ.get('SHELL'): @@ -86,6 +142,8 @@ def shell(bench='.'): def bench(bench='.'): "Bench manager for Frappe" # TODO add bench path context + global FRAPPE_VERSION + FRAPPE_VERSION = get_current_frappe_version() setup_logging(bench=bench) @click.command() @@ -134,8 +192,9 @@ def new_site(site, mariadb_root_password @click.option('--requirements',flag_value=True, type=bool, help="Update requirements") @click.option('--restart-supervisor',flag_value=True, type=bool, help="restart supervisor processes after update") @click.option('--auto',flag_value=True, type=bool) +@click.option('--upgrade',flag_value=True, type=bool) @click.option('--no-backup',flag_value=True, type=bool) -def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False): +def update(pull=False, patch=False, build=False, bench=False, auto=False, restart_supervisor=False, requirements=False, no_backup=False, upgrade=False): "Update bench" if not (pull or patch or build or bench or requirements): @@ -155,12 +214,36 @@ def update(pull=False, patch=False, buil 'build': build, 'requirements': requirements, 'no-backup': no_backup, - 'restart-supervisor': restart_supervisor + 'restart-supervisor': restart_supervisor, + 'upgrade': upgrade }) + + version_upgrade = is_version_upgrade() + + if version_upgrade and not upgrade: + print + print + print "This update will cause a major version change in Frappe/ERPNext from {0} to {1} (beta).".format(*version_upgrade) + print "This would take significant time to migrate and might break custom apps. Please run `bench update --upgrade` to confirm." + print + # print "You can also pin your bench to {0} by running `bench swtich-to-v{0}`".format(version_upgrade[0]) + print "You can stay on the latest stable release by running `bench switch-to-master` or pin your bench to {0} by running `bench swtich-to-v{0}`".format(version_upgrade[0]) + sys.exit(1) + elif not version_upgrade and upgrade: + upgrade = False + if pull: pull_all_apps() + if requirements: update_requirements() + + if upgrade: + pre_upgrade(version_upgrade[0], version_upgrade[1]) + import utils, app + reload(utils) + reload(app) + if patch: if not no_backup: backup_all_sites() @@ -169,14 +252,23 @@ def update(pull=False, patch=False, buil build_assets() if restart_supervisor or conf.get('restart_supervisor_on_update'): restart_supervisor_processes() + if upgrade: + post_upgrade(version_upgrade[0], version_upgrade[1]) print "_"*80 print "https://frappe.io/buy - Donate to help make better free and open source tools" print +@click.command('retry-upgrade') +@click.option('--version', default=5) +def retry_upgrade(version): + pull_all_apps() + patch_sites() + build_assets() + post_upgrade(version-1, version) + def restart_update(kwargs): args = ['--'+k for k, v in kwargs.items() if v] - print 'restarting ' os.execv(sys.argv[0], sys.argv[:2] + args) @click.command('restart') @@ -198,6 +290,33 @@ def migrate_3to4(path): migrate_3to4=os.path.join(os.path.dirname(__file__), 'migrate3to4.py'), site=path)) +@click.command('switch-to-master') +@click.option('--upgrade',flag_value=True, type=bool) +def _switch_to_master(upgrade=False): + "Switch frappe and erpnext to master branch" + switch_to_master(upgrade=upgrade) + print + print 'Switched to master' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + +@click.command('switch-to-develop') +@click.option('--upgrade',flag_value=True, type=bool) +def _switch_to_develop(upgrade=False): + "Switch frappe and erpnext to develop branch" + switch_to_develop(upgrade=upgrade) + print + print 'Switched to develop' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + +@click.command('switch-to-v4') +@click.option('--upgrade',flag_value=True, type=bool) +def _switch_to_v4(upgrade=False): + "Switch frappe and erpnext to v4 branch" + switch_to_v4(upgrade=upgrade) + print + print 'Switched to v4' + print 'Please run `bench update --patch` to be safe from any differences in database schema' + @click.command('set-nginx-port') @click.argument('site') @click.argument('port', type=int) @@ -205,6 +324,27 @@ def set_nginx_port(site, port): "Set nginx port for site" _set_nginx_port(site, port) +@click.command('set-ssl-certificate') +@click.argument('site') +@click.argument('ssl-certificate-path') +def _set_ssl_certificate(site, ssl_certificate_path): + "Set ssl certificate path for site" + set_ssl_certificate(site, ssl_certificate_path) + +@click.command('set-ssl-key') +@click.argument('site') +@click.argument('ssl-certificate-key-path') +def _set_ssl_certificate_key(site, ssl_certificate_key_path): + "Set ssl certificate private key path for site" + set_ssl_certificate_key(site, ssl_certificate_key_path) + +@click.command('set-url-root') +@click.argument('site') +@click.argument('url-root') +def set_url_root(site, url_root): + "Set url root for site" + _set_url_root(site, url_root) + @click.command('set-mariadb-host') @click.argument('host') def _set_mariadb_host(host): @@ -239,11 +379,13 @@ def _prime_wheel_cache(): @click.command('release') @click.argument('app', type=click.Choice(['frappe', 'erpnext', 'shopping_cart'])) @click.argument('bump-type', type=click.Choice(['major', 'minor', 'patch'])) -def _release(app, bump_type): +@click.option('--develop', default='develop') +@click.option('--master', default='master') +def _release(app, bump_type, develop, master): "Release app (internal to the Frappe team)" from .release import release repo = os.path.join('apps', app) - release(repo, bump_type) + release(repo, bump_type, develop, master) ## Setup @click.group() @@ -267,6 +409,11 @@ def setup_supervisor(): "generate config for supervisor" generate_supervisor_config() +@click.command('redis-cache') +def setup_redis_cache(): + "generate config for redis cache" + generate_redis_config() + @click.command('production') @click.argument('user') def setup_production(user): @@ -305,6 +452,7 @@ def setup_config(): setup.add_command(setup_nginx) setup.add_command(setup_sudoers) setup.add_command(setup_supervisor) +setup.add_command(setup_redis_cache) setup.add_command(setup_auto_update) setup.add_command(setup_dnsmasq) setup.add_command(setup_backups) @@ -380,40 +528,32 @@ config.add_command(config_http_timeout) def patch(): pass -@click.command('fix-perms') -def _fix_perms(): +@click.command('fix-prod-perms') +def _fix_prod_perms(): + "Fix permissions if supervisor processes were run as root" if os.path.exists("config/supervisor.conf"): exec_cmd("supervisorctl stop frappe:") - "Fix permissions if supervisor processes were run as root" - files = [ - "logs/web.error.log", - "logs/web.log", - "logs/workerbeat.error.log", - "logs/workerbeat.log", - "logs/worker.error.log", - "logs/worker.log", - "config/nginx.conf", - "config/supervisor.conf", - ] - - frappe_user = get_config().get('frappe_user') - if not frappe_user: - print "frappe user not set" - sys.exit(1) - - for path in files: - if os.path.exists(path): - uid = pwd.getpwnam(frappe_user).pw_uid - gid = grp.getgrnam(frappe_user).gr_gid - os.chown(path, uid, gid) + fix_prod_setup_perms() if os.path.exists("config/supervisor.conf"): exec_cmd("{bench} setup supervisor".format(bench=sys.argv[0])) exec_cmd("supervisorctl reload") -patch.add_command(_fix_perms) +@click.command('fix-file-perms') +def _fix_file_perms(): + "Fix file permissions" + fix_file_perms() + +patch.add_command(_fix_file_perms) +patch.add_command(_fix_prod_perms) + + +@click.command('download-translations') +def _download_translations(): + "Download latest translations" + download_translations_p() #Bench commands @@ -427,12 +567,20 @@ bench.add_command(restart) bench.add_command(config) bench.add_command(start) bench.add_command(set_nginx_port) +bench.add_command(_set_ssl_certificate) +bench.add_command(_set_ssl_certificate_key) bench.add_command(_set_mariadb_host) bench.add_command(set_default_site) bench.add_command(migrate_3to4) +bench.add_command(_switch_to_master) +bench.add_command(_switch_to_develop) +bench.add_command(_switch_to_v4) bench.add_command(shell) bench.add_command(_backup_all_sites) bench.add_command(_backup_site) bench.add_command(_prime_wheel_cache) bench.add_command(_release) bench.add_command(patch) +bench.add_command(set_url_root) +bench.add_command(retry_upgrade) +bench.add_command(_download_translations) --- bench/config.py.orig 2014-11-19 06:36:44 UTC +++ bench/config.py @@ -1,12 +1,27 @@ import os import getpass import json +import subprocess +import shutil from jinja2 import Environment, PackageLoader -from .utils import get_sites, get_config, update_config +from .utils import get_sites, get_config, update_config, get_redis_version env = Environment(loader=PackageLoader('bench', 'templates'), trim_blocks=True) +def write_config_file(bench, file_name, config): + config_path = os.path.join(bench, 'config') + file_path = os.path.join(config_path, file_name) + number = (len([path for path in os.listdir(config_path) if path.startswith(file_name)]) -1 ) or '' + if number: + number = '.' + str(number) + if os.path.exists(file_path): + shutil.move(file_path, file_path + '.save' + number) + + with open(file_path, 'wb') as f: + f.write(config) + def generate_supervisor_config(bench='.', user=None): + from .app import get_current_frappe_version template = env.get_template('supervisor.conf') bench_dir = os.path.abspath(bench) sites_dir = os.path.join(bench_dir, "sites") @@ -20,9 +35,11 @@ def generate_supervisor_config(bench='.' "sites_dir": sites_dir, "user": user, "http_timeout": config.get("http_timeout", 120), + "redis_server": subprocess.check_output('which redis-server', shell=True).strip(), + "redis_config": os.path.join(bench_dir, 'config', 'redis.conf'), + "frappe_version": get_current_frappe_version() }) - with open("config/supervisor.conf", 'w') as f: - f.write(config) + write_config_file(bench, 'supervisor.conf', config) update_config({'restart_supervisor_on_update': True}) def get_site_config(site, bench='.'): @@ -31,10 +48,16 @@ def get_site_config(site, bench='.'): def get_sites_with_config(bench='.'): sites = get_sites() - return [{ - "name": site, - "port": get_site_config(site, bench=bench).get('nginx_port') - } for site in sites] + ret = [] + for site in sites: + site_config = get_site_config(site, bench=bench) + ret.append({ + "name": site, + "port": site_config.get('nginx_port'), + "ssl_certificate": site_config.get('ssl_certificate'), + "ssl_certificate_key": site_config.get('ssl_certificate_key') + }) + return ret def generate_nginx_config(bench='.'): template = env.get_template('nginx.conf') @@ -59,5 +82,14 @@ def generate_nginx_config(bench='.'): "dns_multitenant": get_config().get('dns_multitenant'), "sites": sites }) - with open("config/nginx.conf", 'w') as f: - f.write(config) + write_config_file(bench, 'nginx.conf', config) + +def generate_redis_config(bench='.'): + template = env.get_template('redis.conf') + conf = { + "maxmemory": get_config().get('cache_maxmemory', '50'), + "port": get_config().get('redis_cache_port', '11311'), + "redis_version": get_redis_version() + } + config = template.render(**conf) + write_config_file(bench, 'redis.conf', config) --- bench/migrate_to_v5.py.orig 2015-07-31 10:19:27 UTC +++ bench/migrate_to_v5.py @@ -0,0 +1,46 @@ +from .utils import exec_cmd, get_frappe, run_frappe_cmd +from .release import get_current_version +from .app import remove_from_appstxt +import os +import shutil +import sys + +repos = ('frappe', 'erpnext') + +def migrate_to_v5(bench='.'): + validate_v4(bench=bench) + for repo in repos: + checkout_v5(repo, bench=bench) + remove_shopping_cart(bench=bench) + exec_cmd("{bench} update".format(bench=sys.argv[0])) + +def remove_shopping_cart(bench='.'): + archived_apps_dir = os.path.join(bench, 'archived_apps') + shopping_cart_dir = os.path.join(bench, 'apps', 'shopping_cart') + + if not os.path.exists(shopping_cart_dir): + return + + run_frappe_cmd('--site', 'all', 'remove-from-installed-apps', 'shopping_cart', bench=bench) + remove_from_appstxt('shopping_cart', bench=bench) + exec_cmd("{pip} --no-input uninstall -y shopping_cart".format(pip=os.path.join(bench, 'env', 'bin', 'pip'))) + + if not os.path.exists(archived_apps_dir): + os.mkdir(archived_apps_dir) + shutil.move(shopping_cart_dir, archived_apps_dir) + +def validate_v4(bench='.'): + for repo in repos: + path = os.path.join(bench, 'apps', repo) + if os.path.exists(path): + current_version = get_current_version(path) + if not current_version.startswith('4'): + raise Exception("{} is not on v4.x.x".format(repo)) + +def checkout_v5(repo, bench='.'): + cwd = os.path.join(bench, 'apps', repo) + if os.path.exists(cwd): + exec_cmd("git fetch upstream", cwd=cwd) + exec_cmd("git checkout v5.0", cwd=cwd) + exec_cmd("git clean -df", cwd=cwd) + --- bench/production_setup.py.orig 2014-11-19 06:36:44 UTC +++ bench/production_setup.py @@ -1,17 +1,16 @@ -from .utils import get_program, exec_cmd, get_cmd_output +from .utils import get_program, exec_cmd, get_cmd_output, fix_prod_setup_perms from .config import generate_nginx_config, generate_supervisor_config from jinja2 import Environment, PackageLoader import os import shutil def restart_service(service): - program = get_program(['systemctl', 'service']) - if not program: + if os.path.basename(get_program(['systemctl']) or '') == 'systemctl' and is_running_systemd(): + exec_cmd("{prog} restart {service}".format(prog='systemctl', service=service)) + elif os.path.basename(get_program(['service']) or '') == 'service': + exec_cmd("{prog} {service} restart ".format(prog='service', service=service)) + else: raise Exception, 'No service manager found' - elif os.path.basename(program) == 'systemctl': - exec_cmd("{prog} restart {service}".format(prog=program, service=service)) - elif os.path.basename(program) == 'service': - exec_cmd("{prog} {service} restart ".format(prog=program, service=service)) def get_supervisor_confdir(): possiblities = ('/etc/supervisor/conf.d', '/etc/supervisor.d/', '/etc/supervisord/conf.d', '/etc/supervisord.d') @@ -30,6 +29,14 @@ def remove_default_nginx_configs(): def is_centos7(): return os.path.exists('/etc/redhat-release') and get_cmd_output("cat /etc/redhat-release | sed 's/Linux\ //g' | cut -d' ' -f3 | cut -d. -f1").strip() == '7' +def is_running_systemd(): + with open('/proc/1/comm') as f: + comm = f.read().strip() + if comm == "init": + return False + elif comm == "systemd": + return True + return False def copy_default_nginx_config(): shutil.copy(os.path.join(os.path.dirname(__file__), 'templates', 'nginx_default.conf'), '/etc/nginx/nginx.conf') @@ -37,6 +44,7 @@ def copy_default_nginx_config(): def setup_production(user, bench='.'): generate_supervisor_config(bench=bench, user=user) generate_nginx_config(bench=bench) + fix_prod_setup_perms(frappe_user=user) remove_default_nginx_configs() if is_centos7(): --- bench/release.py.orig 2014-11-19 06:36:44 UTC +++ bench/release.py @@ -34,10 +34,10 @@ def create_release(repo_path, version, r g.merge(master_branch) return tag_name -def push_release(repo_path): +def push_release(repo_path, develop_branch='develop', master_branch='master'): repo = git.Repo(repo_path) g = repo.git - print g.push('upstream', 'master:master', 'develop:develop', '--tags') + print g.push('upstream', '{master}:{master}'.format(master=master_branch), '{develop}:{develop}'.format(develop=develop_branch), '--tags') def create_github_release(owner, repo, tag_name, log, gh_username=None, gh_password=None): global github_username, github_password @@ -137,25 +137,40 @@ def get_current_version(repo): contents) return match.group(2) -def bump_repo(repo, bump_type): - update_branch(repo, 'master', remote='upstream') - update_branch(repo, 'develop', remote='upstream') - git.Repo(repo).git.checkout('develop') - current_version = get_current_version(repo) - new_version = get_bumped_version(current_version, bump_type) - set_version(repo, new_version) - return new_version +def check_for_unmerged_changelog(repo): + current = os.path.join(repo, os.path.basename(repo), 'change_log', 'current') + if os.path.exists(current) and [f for f in os.listdir(current) if f != "readme.md"]: + raise Exception("Unmerged change log! in " + repo) -def bump(repo, bump_type): +def bump_repo(repo, bump_type, develop='develop', master='master', remote='upstream'): + update_branch(repo, master, remote=remote) + update_branch(repo, develop, remote=remote) + git.Repo(repo).git.checkout(develop) + check_for_unmerged_changelog(repo) + current_version = get_current_version(repo) + new_version = get_bumped_version(current_version, bump_type) + set_version(repo, new_version) + return new_version + +def get_release_message(repo_path, develop_branch='develop', master_branch='master'): + repo = git.Repo(repo_path) + g = repo.git + return "* " + g.log('upstream/{master_branch}..upstream/{develop_branch}'.format(master_branch=master_branch, develop_branch=develop_branch), '--format=format:%s', '--no-merges').replace('\n', '\n* ') + +def bump(repo, bump_type, develop='develop', master='master', remote='upstream'): assert bump_type in ['minor', 'major', 'patch'] - new_version = bump_repo(repo, bump_type) + new_version = bump_repo(repo, bump_type, develop=develop, master=master, remote=remote) + message = get_release_message(repo, develop_branch=develop, master_branch=master) + print + print message + print commit_changes(repo, new_version) - tag_name = create_release(repo, new_version) - push_release(repo) - create_github_release('frappe', repo, tag_name, '') + tag_name = create_release(repo, new_version, develop_branch=develop, master_branch=master) + push_release(repo, develop_branch=develop, master_branch=master) + create_github_release('frappe', repo, tag_name, message) print 'Released {tag} for {repo}'.format(tag=tag_name, repo=repo) -def release(repo, bump_type): +def release(repo, bump_type, develop, master): if not get_config().get('release_bench'): print 'bench not configured to release' sys.exit(1) @@ -164,7 +179,7 @@ def release(repo, bump_type): github_password = getpass.getpass() r = requests.get('https://api.github.com/user', auth=HTTPBasicAuth(github_username, github_password)) r.raise_for_status() - bump(repo, bump_type) + bump(repo, bump_type, develop=develop, master=master) if __name__ == "__main__": main() --- bench/templates/nginx.conf.orig 2014-11-19 06:36:44 UTC +++ bench/templates/nginx.conf @@ -5,15 +5,7 @@ upstream frappe { server 127.0.0.1:8000 fail_timeout=0; } -{% macro server_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%} - server { - listen {{ site.port if not default and site.port else port }} {% if default %} default {% endif %}; - client_max_body_size 4G; - {% if dns_multitenant and sites %} - server_name {% for site in sites %} {{ site.name }} {% endfor %}; - {% else %} - server_name {{ site.name if not server_name else server_name }}; - {% endif %} +{% macro location_block(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%} keepalive_timeout 5; sendfile on; root {{ sites_dir }}; @@ -34,30 +26,66 @@ upstream frappe { location @magic { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; {% if not dns_multitenant %} - proxy_set_header Host {{ site.name }}; - {% else %} - proxy_set_header Host $host; + proxy_set_header X-Frappe-Site-Name {{ site.name }}; {% endif %} + proxy_set_header Host $host; proxy_set_header X-Use-X-Accel-Redirect True; proxy_read_timeout {{http_timeout}}; proxy_redirect off; proxy_pass http://frappe; } +{%- endmacro %} + +{% macro server_name_block(site, default=False, server_name=None, sites=None, dns_multitenant=False) -%} + client_max_body_size 4G; + {% if dns_multitenant and sites %} + server_name {% for site in sites %} {{ site.name }} {% endfor %}; + {% else %} + server_name {{ site.name if not server_name else server_name }}; + {% endif %} +{%- endmacro %} + +{% macro server_block_http(site, port=80, default=False, server_name=None, sites=None, dns_multitenant=False) -%} + server { + listen {{ site.port if not default and site.port else port }} {% if default %} default {% endif %}; + {{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} + {{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} + } +{%- endmacro %} + +{% macro server_block_https(site, port=443, default=False, server_name=None, sites=None, dns_multitenant=False) -%} + server { + listen {{ site.ssl_port if not default and site.ssl_port else port }} {% if default %} default {% endif %}; + {{ server_name_block(site, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} + + ssl on; + ssl_certificate {{ site.ssl_certificate }}; + ssl_certificate_key {{ site.ssl_certificate_key }}; + ssl_session_timeout 5m; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + ssl_prefer_server_ciphers on; + + {{ location_block(site, port=port, default=default, server_name=server_name, sites=sites, dns_multitenant=dns_multitenant) }} } {%- endmacro %} {% for site in sites %} {% if site.port %} -{{ server_block(site) }} +{{ server_block_http(site) }} +{% endif %} + +{% if site.ssl_certificate_key and site.ssl_certificate %} +{{ server_block_https(site) }} {% endif %} {% endfor %} {% if default_site %} -{{ server_block(default_site, default=True, server_name="frappe_default_site") }} +{{ server_block_http(default_site, default=True, server_name="frappe_default_site") }} {% endif %} {% if dns_multitenant and sites %} -{{ server_block(None, default=False, sites=sites, dns_multitenant=True) }} +{{ server_block_http(None, default=False, sites=sites, dns_multitenant=True) }} {% endif %} --- bench/templates/redis.conf.orig 2015-07-31 10:19:27 UTC +++ bench/templates/redis.conf @@ -0,0 +1,72 @@ +activerehashing yes +appendfsync everysec +appendonly no +auto-aof-rewrite-min-size 64mb +auto-aof-rewrite-percentage 100 +daemonize no +databases 16 +dbfilename dump.rdb +list-max-ziplist-entries 512 +list-max-ziplist-value 64 +no-appendfsync-on-rewrite no +pidfile /var/run/redis.pid +port {{port}} +rdbcompression yes +set-max-intset-entries 512 +slave-serve-stale-data yes +slowlog-log-slower-than 10000 +slowlog-max-len 128 +timeout 0 +zset-max-ziplist-entries 128 +zset-max-ziplist-value 64 + +maxmemory {{maxmemory}}mb +maxmemory-policy allkeys-lru + +{% if redis_version == "2.4"%} +hash-max-zipmap-entries 512 +hash-max-zipmap-value 64 +loglevel verbose +vm-enabled no +vm-max-memory 0 +vm-max-threads 4 +vm-page-size 32 +vm-pages 134217728 +vm-swap-file /tmp/redis.swap +{% endif %} + +{% if redis_version == "2.6"%} +aof-rewrite-incremental-fsync yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit pubsub 32mb 8mb 60 +client-output-buffer-limit slave 256mb 64mb 60 +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 +hz 10 +loglevel notice +lua-time-limit 5000 +rdbchecksum yes +repl-disable-tcp-nodelay no +slave-read-only yes +stop-writes-on-bgsave-error yes +tcp-keepalive 0 +{% endif %} + +{% if redis_version == "2.8"%} +aof-rewrite-incremental-fsync yes +appendfilename "appendonly.aof" +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit pubsub 32mb 8mb 60 +client-output-buffer-limit slave 256mb 64mb 60 +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 +hz 10 +logfile "" +loglevel notice +lua-time-limit 5000 +notify-keyspace-events "" +rdbchecksum yes +slave-read-only yes +stop-writes-on-bgsave-error yes +tcp-keepalive 0 +{% endif %} --- bench/templates/supervisor.conf.orig 2014-11-19 06:36:44 UTC +++ bench/templates/supervisor.conf @@ -28,5 +28,18 @@ stderr_logfile={{ bench_dir }}/logs/work user={{ user }} directory={{ sites_dir }} + +{% if frappe_version > 4%} +[program:redis-cache] +command={{ redis_server }} {{ redis_config }} +autostart=true +autorestart=true +stopsignal=QUIT +stdout_logfile={{ bench_dir }}/logs/redis.log +stderr_logfile={{ bench_dir }}/logs/redis.error.log +user={{ user }} +directory={{ sites_dir }} +{% endif %} + [group:frappe] programs=frappe-web,frappe-worker,frappe-workerbeat --- bench/utils.py.orig 2014-11-19 06:36:44 UTC +++ bench/utils.py @@ -1,14 +1,24 @@ import os +import re import sys import subprocess import getpass import logging +import itertools +import requests import json +import platform +import multiprocessing from distutils.spawn import find_executable import pwd, grp + +class PatchError(Exception): + pass + logger = logging.getLogger(__name__) + default_config = { 'restart_supervisor_on_update': False, 'auto_update': False, @@ -20,15 +30,20 @@ default_config = { } def get_frappe(bench='.'): - frappe = os.path.abspath(os.path.join(bench, 'env', 'bin', 'frappe')) + frappe = get_env_cmd('frappe', bench=bench) if not os.path.exists(frappe): print 'frappe app is not installed. Run the following command to install frappe' print 'bench get-app frappe https://github.com/frappe/frappe.git' return frappe +def get_env_cmd(cmd, bench='.'): + return os.path.abspath(os.path.join(bench, 'env', 'bin', cmd)) + def init(path, apps_path=None, no_procfile=False, no_backups=False, no_auto_update=False, frappe_path=None, frappe_branch=None, wheel_cache_dir=None): from .app import get_app, install_apps_from_path + from .config import generate_redis_config + global FRAPPE_VERSION if os.path.exists(path): print 'Directory {} already exists!'.format(path) sys.exit(1) @@ -44,9 +59,10 @@ def init(path, apps_path=None, no_procfi if wheel_cache_dir: update_config({"wheel_cache_dir":wheel_cache_dir}, bench=path) prime_wheel_cache(bench=path) + if not frappe_path: frappe_path = 'https://github.com/frappe/frappe.git' - get_app('frappe', frappe_path, branch=frappe_branch, bench=path) + get_app('frappe', frappe_path, branch=frappe_branch, bench=path, build_asset_files=False) if not no_procfile: setup_procfile(bench=path) if not no_backups: @@ -55,6 +71,9 @@ def init(path, apps_path=None, no_procfi setup_auto_update(bench=path) if apps_path: install_apps_from_path(apps_path, bench=path) + FRAPPE_VERSION = get_current_frappe_version(bench=path) + build_assets(bench=path) + generate_redis_config(bench=path) def exec_cmd(cmd, cwd='.'): try: @@ -69,18 +88,31 @@ def setup_env(bench='.'): exec_cmd('./env/bin/pip -q install https://github.com/frappe/MySQLdb1/archive/MySQLdb-1.2.5-patched.tar.gz', cwd=bench) def setup_procfile(bench='.'): + from .app import get_current_frappe_version + frappe_version = get_current_frappe_version() + procfile_contents = { + 'web': "./env/bin/frappe --serve --sites_path sites", + 'worker': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker'", + 'workerbeat': "sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'" + } + if frappe_version > 4: + procfile_contents['redis_cache'] = "redis-server config/redis.conf" + procfile_contents['web'] = "bench serve" + + procfile = '\n'.join(["{0}: {1}".format(k, v) for k, v in procfile_contents.items()]) + with open(os.path.join(bench, 'Procfile'), 'w') as f: - f.write("""web: ./env/bin/frappe --serve --sites_path sites -worker: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app worker' -workerbeat: sh -c 'cd sites && exec ../env/bin/python -m frappe.celery_app beat -s scheduler.schedule'""") + f.write(procfile) def new_site(site, mariadb_root_password=None, admin_password=None, bench='.'): + import hashlib logger.info('creating new site {}'.format(site)) mariadb_root_password_fragment = '--root_password {}'.format(mariadb_root_password) if mariadb_root_password else '' admin_password_fragment = '--admin_password {}'.format(admin_password) if admin_password else '' - exec_cmd("{frappe} --install {site} {site} {mariadb_root_password_fragment} {admin_password_fragment}".format( + exec_cmd("{frappe} {site} --install {db_name} {mariadb_root_password_fragment} {admin_password_fragment}".format( frappe=get_frappe(bench=bench), site=site, + db_name = hashlib.sha1(site).hexdigest()[:10], mariadb_root_password_fragment=mariadb_root_password_fragment, admin_password_fragment=admin_password_fragment ), cwd=os.path.join(bench, 'sites')) @@ -88,14 +120,23 @@ def new_site(site, mariadb_root_password exec_cmd("{frappe} --use {site}".format(frappe=get_frappe(bench=bench), site=site), cwd=os.path.join(bench, 'sites')) def patch_sites(bench='.'): - exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites')) + try: + if FRAPPE_VERSION == 4: + exec_cmd("{frappe} --latest all".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites')) + else: + run_frappe_cmd('--site', 'all', 'migrate', bench=bench) + except subprocess.CalledProcessError: + raise PatchError def build_assets(bench='.'): - exec_cmd("{frappe} --build".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites')) + if FRAPPE_VERSION == 4: + exec_cmd("{frappe} --build".format(frappe=get_frappe(bench=bench)), cwd=os.path.join(bench, 'sites')) + else: + run_frappe_cmd('build', bench=bench) def get_sites(bench='.'): sites_dir = os.path.join(bench, "sites") - sites = [site for site in os.listdir(sites_dir) + sites = [site for site in os.listdir(sites_dir) if os.path.isdir(os.path.join(sites_dir, site)) and site not in ('assets',)] return sites @@ -115,14 +156,22 @@ def setup_auto_update(bench='.'): def setup_backups(bench='.'): logger.info('setting up backups') - add_to_crontab('0 */6 * * * cd {sites_dir} && {frappe} --backup all >> {logfile} 2>&1'.format(sites_dir=get_sites_dir(bench=bench), - frappe=get_frappe(bench=bench), + bench_dir = get_bench_dir(bench=bench) + if FRAPPE_VERSION == 4: + backup_command = "cd {sites_dir} && {frappe} --backup all".format(frappe=get_frappe(bench=bench),) + else: + backup_command = "cd {bench_dir} && {bench} --site all backup".format(bench_dir=bench_dir, bench=sys.argv[0]) + + add_to_crontab('0 */6 * * * {backup_command} >> {logfile} 2>&1'.format(backup_command=backup_command, logfile=os.path.join(get_bench_dir(bench=bench), 'logs', 'backup.log'))) def add_to_crontab(line): current_crontab = read_crontab() if not line in current_crontab: - s = subprocess.Popen("crontab", stdin=subprocess.PIPE) + cmd = ["crontab"] + if platform.system() == 'FreeBSD': + cmd = ["crontab", "-"] + s = subprocess.Popen(cmd, stdin=subprocess.PIPE) s.stdin.write(current_crontab) s.stdin.write(line + '\n') s.stdin.close() @@ -182,11 +231,12 @@ def get_program(programs): def get_process_manager(): return get_program(['foreman', 'forego', 'honcho']) - + def start(): program = get_process_manager() if not program: raise Exception("No process manager found") + os.environ['PYTHONUNBUFFERED'] = "true" os.execv(program, [program, 'start']) def check_cmd(cmd, cwd='.'): @@ -208,9 +258,10 @@ def check_git_for_shallow_clone(): def get_cmd_output(cmd, cwd='.'): try: - return subprocess.check_output(cmd, cwd=cwd, shell=True) + return subprocess.check_output(cmd, cwd=cwd, shell=True, stderr=open(os.devnull, 'wb')).strip() except subprocess.CalledProcessError, e: - print "Error:", e.output + if e.output: + print e.output raise def restart_supervisor_processes(bench='.'): @@ -236,13 +287,28 @@ def update_site_config(site, new_config, put_site_config(site, config, bench=bench) def set_nginx_port(site, port, bench='.', gen_config=True): + set_site_config_nginx_property(site, {"nginx_port": port}, bench=bench) + +def set_ssl_certificate(site, ssl_certificate, bench='.', gen_config=True): + set_site_config_nginx_property(site, {"ssl_certificate": ssl_certificate}, bench=bench) + +def set_ssl_certificate_key(site, ssl_certificate_key, bench='.', gen_config=True): + set_site_config_nginx_property(site, {"ssl_certificate_key": ssl_certificate_key}, bench=bench) + +def set_nginx_port(site, port, bench='.', gen_config=True): + set_site_config_nginx_property(site, {"nginx_port": port}, bench=bench) + +def set_site_config_nginx_property(site, config, bench='.', gen_config=True): from .config import generate_nginx_config if site not in get_sites(bench=bench): raise Exception("No such site") - update_site_config(site, {"nginx_port": port}, bench=bench) + update_site_config(site, config, bench=bench) if gen_config: generate_nginx_config() +def set_url_root(site, url_root, bench='.'): + update_site_config(site, {"host_name": url_root}, bench=bench) + def set_default_site(site, bench='.'): if not site in get_sites(bench=bench): raise Exception("Site not in bench") @@ -258,8 +324,11 @@ def update_requirements(bench='.'): exec_cmd("{pip} install -q -r {req_file}".format(pip=pip, req_file=req_file)) def backup_site(site, bench='.'): - exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench=bench), site=site), - cwd=os.path.join(bench, 'sites')) + if FRAPPE_VERSION == 4: + exec_cmd("{frappe} --backup {site}".format(frappe=get_frappe(bench=bench), site=site), + cwd=os.path.join(bench, 'sites')) + else: + run_frappe_cmd('--site', site, 'backup', bench=bench) def backup_all_sites(bench='.'): for site in get_sites(bench=bench): @@ -313,4 +382,133 @@ def drop_privileges(uid_name='nobody', g os.setuid(running_uid) # Ensure a very conservative umask - old_umask = os.umask(077) + old_umask = os.umask(022) + +def fix_prod_setup_perms(frappe_user=None): + files = [ + "logs/web.error.log", + "logs/web.log", + "logs/workerbeat.error.log", + "logs/workerbeat.log", + "logs/worker.error.log", + "logs/worker.log", + "config/nginx.conf", + "config/supervisor.conf", + ] + + if not frappe_user: + frappe_user = get_config().get('frappe_user') + + if not frappe_user: + print "frappe user not set" + sys.exit(1) + + for path in files: + if os.path.exists(path): + uid = pwd.getpwnam(frappe_user).pw_uid + gid = grp.getgrnam(frappe_user).gr_gid + os.chown(path, uid, gid) + +def fix_file_perms(): + for dir_path, dirs, files in os.walk('.'): + for _dir in dirs: + os.chmod(os.path.join(dir_path, _dir), 0755) + for _file in files: + os.chmod(os.path.join(dir_path, _file), 0644) + bin_dir = './env/bin' + if os.path.exists(bin_dir): + for _file in os.listdir(bin_dir): + if not _file.startswith('activate'): + os.chmod(os.path.join(bin_dir, _file), 0755) + +def get_redis_version(): + version_string = subprocess.check_output('redis-server --version', shell=True).strip() + if re.search("Redis server version 2.4", version_string): + return "2.4" + if re.search("Redis server v=2.6", version_string): + return "2.6" + if re.search("Redis server v=2.8", version_string): + return "2.8" + +def get_current_frappe_version(bench='.'): + from .app import get_current_frappe_version as fv + return fv(bench=bench) + +def run_frappe_cmd(*args, **kwargs): + bench = kwargs.get('bench', '.') + f = get_env_cmd('python', bench=bench) + sites_dir = os.path.join(bench, 'sites') + subprocess.check_call((f, '-m', 'frappe.utils.bench_helper', 'frappe') + args, cwd=sites_dir) + + +def pre_upgrade(from_ver, to_ver, bench='.'): + from .migrate_to_v5 import validate_v4, remove_shopping_cart + pip = os.path.join(bench, 'env', 'bin', 'pip') + if from_ver == 4 and to_ver == 5: + apps = ('frappe', 'erpnext') + remove_shopping_cart(bench=bench) + + for app in apps: + cwd = os.path.abspath(os.path.join(bench, 'apps', app)) + if os.path.exists(cwd): + exec_cmd("git clean -dxf", cwd=cwd) + exec_cmd("{pip} install --upgrade -e {app}".format(pip=pip, app=cwd)) + +def post_upgrade(from_ver, to_ver, bench='.'): + from .app import get_current_frappe_version + from .config import generate_nginx_config, generate_supervisor_config, generate_redis_config + conf = get_config(bench=bench) + if from_ver == 4 and to_ver == 5: + print "-"*80 + print "Your bench was upgraded to version 5" + if conf.get('restart_supervisor_on_update'): + generate_redis_config(bench=bench) + generate_supervisor_config(bench=bench) + generate_nginx_config(bench=bench) + setup_procfile(bench=bench) + setup_backups(bench=bench) + print "As you have setup your bench for production, you will have to reload configuration for nginx and supervisor" + print "To complete the migration, please run the following commands" + print + print "sudo service nginx restart" + print "sudo supervisorctl reload" + +def update_translations_p(args): + update_translations(*args) + +def download_translations_p(): + pool = multiprocessing.Pool(8) + + langs = get_langs() + apps = ('frappe', 'erpnext') + args = list(itertools.product(apps, langs)) + + pool.map(update_translations_p, args) + +def download_translations(): + langs = get_langs() + apps = ('frappe', 'erpnext') + for app, lang in itertools.product(apps, langs): + update_translations(app, lang) + + +def get_langs(): + lang_file = 'apps/frappe/frappe/data/languages.txt' + with open(lang_file) as f: + lang_data = f.read() + langs = [line.split('\t')[0] for line in lang_data.splitlines()] + langs.remove('en') + return langs + + +def update_translations(app, lang): + translations_dir = os.path.join('apps', app, app, 'translations') + csv_file = os.path.join(translations_dir, lang + '.csv') + r = requests.get("https://translate.erpnext.com/files/{}-{}.csv".format(app, lang)) + r.raise_for_status() + with open(csv_file, 'wb') as f: + f.write(r.text.encode('utf-8')) + print 'downloaded for', app, lang + + +FRAPPE_VERSION = get_current_frappe_version() --- completion.sh.orig 2015-07-31 10:19:27 UTC +++ completion.sh @@ -0,0 +1,30 @@ +_setup_bench_tab_completion () { + if [ -n "$BASH" ] ; then + _bench () { + local cur=${COMP_WORDS[COMP_CWORD]} + local prev=${COMP_WORDS[COMP_CWORD-1]} + if [[ $prev == "--site" ]]; then + COMPREPLY=( $(compgen -W "`_site_dirs`" -- $cur) ) + fi + } + complete -F _bench bench + elif [ -n "$ZSH_VERSION" ]; then + _bench () { + local a + local prev + read -l a + prev=`echo $a| awk '{ print $NF }'` + if [[ $prev == "--site" ]]; then + reply=($(_site_dirs)) + fi + } + compctl -K _bench bench + fi +} + +_site_dirs() { + ls -d sites/*/ | sed "s/sites\///g" | sed "s/\/$//g" | xargs echo +} + + +_setup_bench_tab_completion --- install_scripts/erpnext-apps-master.json.orig 2014-11-19 06:36:44 UTC +++ install_scripts/erpnext-apps-master.json @@ -3,10 +3,5 @@ "url":"https://github.com/frappe/erpnext", "name":"erpnext", "branch": "master" - }, - { - "url":"https://github.com/frappe/shopping-cart", - "name":"shopping_cart", - "branch": "master" } ] --- install_scripts/erpnext-apps.json.orig 2014-11-19 06:36:44 UTC +++ install_scripts/erpnext-apps.json @@ -2,9 +2,5 @@ { "url":"https://github.com/frappe/erpnext", "name":"erpnext" - }, - { - "url":"https://github.com/frappe/shopping-cart", - "name":"shopping_cart" } ] --- install_scripts/setup_frappe.sh.orig 2014-11-19 06:36:44 UTC +++ install_scripts/setup_frappe.sh @@ -16,7 +16,7 @@ get_passwd() { } set_opts () { - OPTS=`getopt -o v --long verbose,mysql-root-password:,frappe-user:,setup-production,help -n 'parse-options' -- "$@"` + OPTS=`getopt -o v --long verbose,mysql-root-password:,frappe-user:,bench-branch:,setup-production,skip-setup-bench,help -n 'parse-options' -- "$@"` if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi @@ -25,10 +25,21 @@ set_opts () { VERBOSE=false HELP=false FRAPPE_USER=false - FRAPPE_USER_PASS=`get_passwd` - MSQ_PASS=`get_passwd` - ADMIN_PASS=`get_passwd` + BENCH_BRANCH="master" SETUP_PROD=false + SETUP_BENCH=true + + if [ -f ~/frappe_passwords.sh ]; then + source ~/frappe_passwords.sh + else + FRAPPE_USER_PASS=`get_passwd` + MSQ_PASS=`get_passwd` + ADMIN_PASS=`get_passwd` + + echo "FRAPPE_USER_PASS=$FRAPPE_USER_PASS" > ~/frappe_passwords.sh + echo "MSQ_PASS=$MSQ_PASS" >> ~/frappe_passwords.sh + echo "ADMIN_PASS=$ADMIN_PASS" >> ~/frappe_passwords.sh + fi while true; do case "$1" in @@ -37,6 +48,8 @@ set_opts () { --mysql-root-password ) MSQ_PASS="$2"; shift; shift ;; --frappe-user ) FRAPPE_USER="$2"; shift; shift ;; --setup-production ) SETUP_PROD=true; shift;; + --bench-branch ) BENCH_BRANCH="$2"; shift;; + --skip-setup-bench ) SETUP_BENCH=false; shift;; -- ) shift; break ;; * ) break ;; esac @@ -94,7 +107,7 @@ add_centos6_mariadb_repo() { echo " [mariadb] name = MariaDB -baseurl = http://yum.mariadb.org/5.5/centos$OS_VER-$ARCH +baseurl = http://yum.mariadb.org/10.0/centos$OS_VER-$ARCH gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB gpgcheck=1 " > /etc/yum.repos.d/mariadb.repo @@ -105,7 +118,7 @@ add_ubuntu_mariadb_repo() { run_cmd sudo apt-get update run_cmd sudo apt-get install -y software-properties-common python-software-properties run_cmd sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db - run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/5.5/ubuntu $OS_VER main" + run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/10.0/ubuntu $OS_VER main" } add_debian_mariadb_repo() { @@ -122,15 +135,15 @@ add_debian_mariadb_repo() { run_cmd sudo apt-get update run_cmd sudo apt-get install -y python-software-properties run_cmd sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db - run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/5.5/debian $CODENAME main" + run_cmd sudo add-apt-repository "deb http://ams2.mirrors.digitalocean.com/mariadb/repo/10.0/debian $CODENAME main" } add_ius_repo() { if [ $OS_VER -eq "6" ]; then wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/epel-release-6-5.noarch.rpm - wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/ius-release-1.0-13.ius.centos6.noarch.rpm + wget http://dl.iuscommunity.org/pub/ius/stable/CentOS/$OS_VER/$T_ARCH/ius-release-1.0-14.ius.centos6.noarch.rpm rpm --quiet -q epel-release || rpm -Uvh epel-release-6-5.noarch.rpm - rpm --quiet -q ius-release || rpm -Uvh ius-release-1.0-13.ius.centos6.noarch.rpm + rpm --quiet -q ius-release || rpm -Uvh ius-release-1.0-14.ius.centos6.noarch.rpm fi } @@ -139,7 +152,9 @@ add_epel_centos7() { } add_maria_db_repo() { - if [ "$OS" == "centos" ]; then + if [ "$OS" == "Ubuntu" ] && [ $OS_VER == "utopic" ]; then + return + elif [ "$OS" == "centos" ]; then echo Adding centos mariadb repo add_centos6_mariadb_repo @@ -148,7 +163,7 @@ add_maria_db_repo() { add_debian_mariadb_repo elif [ "$OS" == "Ubuntu" ]; then - echo Adding debian mariadb repo + echo Adding ubuntu mariadb repo add_ubuntu_mariadb_repo else echo Unsupported Distribution @@ -164,10 +179,10 @@ install_packages() { run_cmd sudo yum groupinstall -y "Development tools" if [ $OS_VER == "6" ]; then run_cmd add_ius_repo - run_cmd sudo yum install -y git MariaDB-server MariaDB-client MariaDB-compat python-setuptools nginx zlib-devel bzip2-devel openssl-devel memcached postfix python27-devel python27 libxml2 libxml2-devel libxslt libxslt-devel redis MariaDB-devel libXrender libXext python27-setuptools + run_cmd sudo yum install -y git MariaDB-server MariaDB-client MariaDB-compat python-setuptools nginx zlib-devel bzip2-devel openssl-devel postfix python27-devel python27 libxml2 libxml2-devel libxslt libxslt-devel redis MariaDB-devel libXrender libXext python27-setuptools cronie sudo which xorg-x11-fonts-Type1 xorg-x11-fonts-75dpi elif [ $OS_VER == "7" ]; then run_cmd add_epel_centos7 - run_cmd sudo yum install -y git mariadb-server mariadb-devel python-setuptools nginx zlib-devel bzip2-devel openssl-devel memcached postfix python-devel libxml2 libxml2-devel libxslt libxslt-devel redis libXrender libXext supervisor + run_cmd sudo yum install -y git mariadb-server mariadb-devel python-setuptools nginx zlib-devel bzip2-devel openssl-devel postfix python-devel libxml2 libxml2-devel libxslt libxslt-devel redis libXrender libXext supervisor cronie sudo which xorg-x11-fonts-75dpi xorg-x11-fonts-Type1 fi echo "Installing wkhtmltopdf" install_wkhtmltopdf_centos @@ -178,7 +193,7 @@ install_packages() { export DEBIAN_FRONTEND=noninteractive setup_debconf run_cmd sudo apt-get update - run_cmd sudo apt-get install python-dev python-setuptools build-essential python-mysqldb git memcached ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx supervisor python-pip fontconfig libxrender1 libxext6 -y + run_cmd sudo apt-get install python-dev python-setuptools build-essential python-mysqldb git ntp vim screen htop mariadb-server mariadb-common libmariadbclient-dev libxslt1.1 libxslt1-dev redis-server libssl-dev libcrypto++-dev postfix nginx supervisor python-pip fontconfig libxrender1 libxext6 xfonts-75dpi xfonts-base -y echo "Installing wkhtmltopdf" install_wkhtmltopdf_deb @@ -190,17 +205,17 @@ install_packages() { install_wkhtmltopdf_centos () { - if [[ $OS == "centos" && $OS_VER == "7" && $T_ARCH="i386" ]]; then + if [[ $OS == "centos" && $OS_VER == "7" && $T_ARCH == "i386" ]]; then echo "Cannot install wkhtmltodpdf. Skipping..." return 0 fi - RPM="wkhtmltox-0.12.1_linux-$OS$OS_VER-$WK_ARCH.rpm" - run_cmd wget http://downloads.sourceforge.net/project/wkhtmltopdf/0.12.1/$RPM + RPM="wkhtmltox-0.12.2.1_linux-$OS$OS_VER-$WK_ARCH.rpm" + run_cmd wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/$RPM rpm --quiet -q wkhtmltox || run_cmd rpm -Uvh $RPM } install_wkhtmltopdf_deb () { - if [[ $OS_VER == "utopic" ]]; then + if [[ $OS_VER == "utopic" || $OS_VER == "vivid" ]]; then echo "Cannot install wkhtmltodpdf. Skipping..." return 0 fi @@ -209,8 +224,8 @@ install_wkhtmltopdf_deb () { else WK_VER=$OS_VER fi - run_cmd wget http://downloads.sourceforge.net/project/wkhtmltopdf/0.12.1/wkhtmltox-0.12.1_linux-$WK_VER-$WK_ARCH.deb - run_cmd dpkg -i wkhtmltox-0.12.1_linux-$WK_VER-$WK_ARCH.deb + run_cmd wget http://download.gna.org/wkhtmltopdf/0.12/0.12.2.1/wkhtmltox-0.12.2.1_linux-$WK_VER-$WK_ARCH.deb + run_cmd dpkg -i wkhtmltox-0.12.2.1_linux-$WK_VER-$WK_ARCH.deb } @@ -274,18 +289,47 @@ configure_services_centos6() { configure_services_centos7() { run_cmd systemctl enable nginx - run_cmd systemctl enable mariadb + run_cmd systemctl enable mysql run_cmd systemctl enable redis run_cmd systemctl enable supervisord - run_cmd systemctl enable memcached } start_services_centos7() { run_cmd systemctl start nginx - run_cmd systemctl start mariadb + run_cmd systemctl start mysql run_cmd systemctl start redis run_cmd systemctl start supervisord - run_cmd systemctl start memcached +} + +configure_mariadb() { + config=" +[mysqld] +innodb-file-format=barracuda +innodb-file-per-table=1 +innodb-large-prefix=1 +character-set-client-handshake = FALSE +character-set-server = utf8mb4 +collation-server = utf8mb4_unicode_ci + +[mysql] +default-character-set = utf8mb4 + " + deb_cnf_path="/etc/mysql/conf.d/barracuda.cnf" + centos_cnf_path="/etc/my.cnf.d/barracuda.cnf" + + if [ $OS == "centos" ]; then + + echo "$config" > $centos_cnf_path + if [ $OS_VER == "6" ]; then + run_cmd sudo service mysql restart + elif [ $OS_VER == "7" ]; then + run_cmd sudo systemctl restart mysql + fi + + elif [ $OS == "debian" ] || [ $OS == "Ubuntu" ]; then + echo "$config" > $deb_cnf_path + sudo service mysql restart + fi } setup_debconf() { @@ -296,14 +340,14 @@ setup_debconf() { } install_bench() { - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER && git clone https://github.com/frappe/bench bench-repo" - if hash pip-2.7; then + run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER && git clone https://github.com/frappe/bench --branch $BENCH_BRANCH bench-repo" + if hash pip-2.7 &> /dev/null; then PIP="pip-2.7" - elif hash pip2.7; then + elif hash pip2.7 &> /dev/null; then PIP="pip2.7" - elif hash pip2; then + elif hash pip2 &> /dev/null; then PIP="pip2" - elif hash pip; then + elif hash pip &> /dev/null; then PIP="pip" else echo PIP not installed @@ -325,12 +369,12 @@ setup_bench() { echo Setting up first site echo /home/$FRAPPE_USER/frappe-bench > /etc/frappe_bench_dir run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench new-site site1.local --mariadb-root-password $MSQ_PASS --admin-password $ADMIN_PASS" - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench frappe --install_app erpnext" - run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench frappe --install_app shopping_cart" + run_cmd sudo su $FRAPPE_USER -c "cd /home/$FRAPPE_USER/frappe-bench && bench install-app erpnext" run_cmd bash -c "cd /home/$FRAPPE_USER/frappe-bench && bench setup sudoers $FRAPPE_USER" if $SETUP_PROD; then run_cmd bash -c "cd /home/$FRAPPE_USER/frappe-bench && bench setup production $FRAPPE_USER" fi + chown $FRAPPE_USER /home/$FRAPPE_USER/frappe-bench/logs/* } add_user() { @@ -377,10 +421,13 @@ main() { fi configure_mariadb_centos fi + configure_mariadb echo "Adding frappe user" add_user install_bench - setup_bench + if $SETUP_BENCH; then + setup_bench + fi echo RUNNING="" --- setup.py.orig 2014-11-19 06:36:44 UTC +++ setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packa setup( name='bench', - version='0.1', + version='0.92', py_modules=find_packages(), include_package_data=True, url='https://github.com/frappe/bench',