source: rattail/rattail/commands/core.py @ 80130f3

Last change on this file since 80130f3 was 80130f3, checked in by Lance Edgar <ledgar@…>, 9 months ago

Add debug logging when "stale changes" detected for datasync

printing to stderr is useful for shinken, but need to be able to reference same
events within the log file

  • Property mode set to 100644
File size: 49.8 KB
Line 
1# -*- coding: utf-8; -*-
2################################################################################
3#
4#  Rattail -- Retail Software Framework
5#  Copyright © 2010-2019 Lance Edgar
6#
7#  This file is part of Rattail.
8#
9#  Rattail is free software: you can redistribute it and/or modify it under the
10#  terms of the GNU General Public License as published by the Free Software
11#  Foundation, either version 3 of the License, or (at your option) any later
12#  version.
13#
14#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
15#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
16#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
17#  details.
18#
19#  You should have received a copy of the GNU General Public License along with
20#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
21#
22################################################################################
23"""
24Console Commands
25"""
26
27from __future__ import unicode_literals, absolute_import
28
29import os
30import sys
31import time
32import platform
33import argparse
34import datetime
35import socket
36import shutil
37import subprocess
38import warnings
39import logging
40from getpass import getpass
41
42import six
43import humanize
44
45from rattail import __version__
46from rattail.core import Object
47from rattail.util import load_entry_points, load_object
48from rattail.progress import ConsoleProgress, SocketProgress
49from rattail.config import make_config, parse_list
50from rattail.util import progress_loop
51from rattail.time import make_utc
52from rattail.db.config import configure_versioning
53
54
55log = logging.getLogger(__name__)
56
57
58class ArgumentParser(argparse.ArgumentParser):
59    """
60    Custom argument parser.
61
62    This overrides some of the parsing logic which is specific to the primary
63    command object.
64    """
65
66    def parse_args(self, args=None, namespace=None):
67        args, argv = self.parse_known_args(args, namespace)
68        args.argv = argv
69        return args
70
71
72def date_argument(string):
73    """
74    Validate and coerce a date argument.
75
76    This function is designed be used as the ``type`` parameter when calling
77    ``ArgumentParser.add_argument()``, e.g.::
78
79       parser = ArgumentParser()
80       parser.add_argument('--date', type=date_argument)
81    """
82    try:
83        date = datetime.datetime.strptime(string, '%Y-%m-%d').date()
84    except ValueError:
85        raise argparse.ArgumentTypeError("Date must be in YYYY-MM-DD format")
86    return date
87
88
89def list_argument(string):
90    """
91    Coerce the given string to a list of strings, splitting on whitespace
92    and/or commas.
93
94    This function is designed be used as the ``type`` parameter when calling
95    ``ArgumentParser.add_argument()``, e.g.::
96
97       parser = ArgumentParser()
98       parser.add_argument('--things', type=list_argument)
99    """
100    return parse_list(string)
101
102
103@six.python_2_unicode_compatible
104class Command(Object):
105    """
106    The primary command for the application.
107
108    This effectively *is* the ``rattail`` console application.  It mostly
109    provides the structure for subcommands, which really do all the work.
110
111    This command is designed to be subclassed, should your application need
112    similar functionality.
113    """
114    name = 'rattail'
115    version = __version__
116    description = "Rattail Software Framework"
117    long_description = """
118Rattail is a retail software framework.
119
120Copyright (c) 2010-2015 Lance Edgar <lance@edbob.org>
121
122This program comes with ABSOLUTELY NO WARRANTY.  This is free software,
123and you are welcome to redistribute it under certain conditions.
124See the file COPYING.txt for more information.
125"""
126
127    stdout = sys.stdout
128    stderr = sys.stderr
129
130    def __init__(self, **kwargs):
131        super(Command, self).__init__(**kwargs)
132        self.subcommands = load_entry_points('{}.commands'.format(self.name.replace('-', '_')))
133
134    def __str__(self):
135        return self.name
136
137    @property
138    def db_config_section(self):
139        """
140        Name of section in config file which should have database connection
141        info.  This defaults to ``'rattail.db'`` but may be overridden so the
142        command framework can sit in front of a non-Rattail database if needed.
143
144        This is used to auto-configure a "default" database engine for the app,
145        when any command is invoked.
146        """
147        return 'rattail.db'
148
149    @property
150    def db_session_factory(self):
151        """
152        Reference to the "primary" ``Session`` class, which will be configured
153        automatically during app startup.  Defaults to :class:`rattail.db.Session`.
154        """
155        from rattail.db import Session
156        return Session
157
158    @property
159    def db_model(self):
160        """
161        Reference to the Python module which is to be used as the primary data
162        model.  Defaults to ``rattail.db.model``.
163        """
164        return self.config.get_model()
165
166    def iter_subcommands(self):
167        """
168        Iterate over the subcommands.
169
170        This is a generator which yields each associated :class:`Subcommand`
171        class sorted by :attr:`Subcommand.name`.
172        """
173        for name in sorted(self.subcommands):
174            yield self.subcommands[name]
175
176    def print_help(self):
177        """
178        Print help text for the primary command.
179
180        The output will include a list of available subcommands.
181        """
182        # TODO: this should leverage parser config...
183        self.stdout.write("""{0.description}
184
185Usage: {0.name} [options] <command> [command-options]
186
187Options:
188  -c PATH, --config=PATH
189                    Config path (may be specified more than once)
190  -P, --progress    Show progress indicators (where relevant)
191  -R, --runas       Optional username to impersonate when running command
192  -V, --version     Display program version and exit
193
194Commands:\n""".format(self))
195
196        for cmd in self.iter_subcommands():
197            self.stdout.write("  {0:<16s}  {1}\n".format(cmd.name, cmd.description))
198
199        self.stdout.write("\nTry '{0} help <command>' for more help.\n".format(self.name))
200
201    def run(self, *args):
202        """
203        Parse command line arguments and execute appropriate subcommand.
204        """
205        parser = ArgumentParser(
206            prog=self.name,
207            description=self.description,
208            add_help=False,
209            )
210
211        parser.add_argument('-c', '--config', action='append', dest='config_paths',
212                            metavar='PATH')
213
214        # TODO: i think these aren't really being used in practice..?
215        parser.add_argument('-n', '--no-init', action='store_true', default=False)
216        parser.add_argument('--no-extend-config', dest='extend_config', action='store_false')
217
218        parser.add_argument('--verbose', action='store_true')
219        parser.add_argument('-P', '--progress', action='store_true', default=False)
220        parser.add_argument('--progress-socket',
221                            help="Optional socket (e.g. localhost:8487) to which progress info should be written.")
222        parser.add_argument('--runas', '-R', metavar='USERNAME',
223                            help="Optional username to impersonate when running the command.  "
224                            "This is only relevant for / used by certain commands.")
225        parser.add_argument('--stdout', metavar='PATH', type=argparse.FileType('w'),
226                            help="Optional path to which STDOUT should be effectively redirected.")
227        parser.add_argument('--stderr', metavar='PATH', type=argparse.FileType('w'),
228                            help="Optional path to which STDERR should be effectively redirected.")
229
230        # data versioning
231        parser.add_argument('--versioning', action='store_true',
232                            help="Force *enable* of data versioning.  If set, then --no-versioning "
233                            "cannot also be set.  If neither is set, config will determine whether "
234                            "or not data versioning should be enabled.")
235        parser.add_argument('--no-versioning', action='store_true',
236                            help="Force *disable* of data versioning.  If set, then --versioning "
237                            "cannot also be set.  If neither is set, config will determine whether "
238                            "or not data versioning should be enabled.")
239
240        parser.add_argument('-V', '--version', action='version',
241                            version="%(prog)s {0}".format(self.version))
242        parser.add_argument('command', nargs='*')
243
244        # Parse args and determine subcommand.
245        args = parser.parse_args(list(args))
246        if not args or not args.command:
247            self.print_help()
248            return
249
250        # TODO: can we make better args so this is handled by argparse somehow?
251        if args.versioning and args.no_versioning:
252            sys.stderr.write("Cannot pass both --versioning and --no-versioning\n")
253            sys.exit(1)
254
255        # Show (sub)command help if so instructed, or unknown subcommand.
256        cmd = args.command.pop(0)
257        if cmd == 'help':
258            if len(args.command) != 1:
259                self.print_help()
260                return
261            cmd = args.command[0]
262            if cmd not in self.subcommands:
263                self.print_help()
264                return
265            cmd = self.subcommands[cmd](parent=self)
266            cmd.parser.print_help()
267            return
268        elif cmd in self.subcommands:
269            if '-h' in args.argv or '--help' in args.argv:
270                cmd = self.subcommands[cmd](parent=self)
271                cmd.parser.print_help()
272                return
273        else:
274            self.print_help()
275            return
276
277        # Okay, we should be done needing to print help messages.  Now it's
278        # safe to redirect STDOUT/STDERR, if necessary.
279        if args.stdout:
280            self.stdout = args.stdout
281        if args.stderr:
282            self.stderr = args.stderr
283
284        # if args say not to "init" then we make a sort of empty config
285        if args.no_init:
286            self.config = make_config([], extend=False, versioning=False)
287
288        else: # otherwise we make a proper config, and maybe turn on versioning
289            logging.basicConfig()
290            self.config = make_config(args.config_paths, extend=args.extend_config, versioning=False)
291            if args.versioning:
292                configure_versioning(self.config, force=True)
293            elif not args.no_versioning:
294                configure_versioning(self.config)
295
296        # instantiate the subcommand
297        subcmd = self.subcommands[cmd](self, self.config)
298
299        # figure out if/how subcommand should show progress
300        subcmd.show_progress = args.progress
301        subcmd.progress = None
302        if subcmd.show_progress:
303            if args.progress_socket:
304                host, port = args.progress_socket.split(':')
305                subcmd.progress = SocketProgress(host, int(port))
306            else:
307                subcmd.progress = ConsoleProgress
308
309        # maybe should be verbose
310        subcmd.verbose = args.verbose
311
312        # TODO: make this default to something from config?
313        subcmd.runas_username = args.runas or None
314
315        # and finally, run the subcommand
316        log.debug("running '%s %s' with args: %s", self.name, subcmd.name, args.argv)
317        subcmd._run(*(args.command + args.argv))
318
319
320class Subcommand(object):
321    """
322    Base class for application subcommands.
323    """
324    name = 'UNDEFINED'
325    description = 'UNDEFINED'
326
327    def __init__(self, parent=None, config=None, show_progress=None):
328        self.parent = parent
329        self.config = config
330        self.stdout = getattr(parent, 'stdout', sys.stdout)
331        self.stderr = getattr(parent, 'stderr', sys.stderr)
332        self.show_progress = show_progress
333        self.progress = ConsoleProgress if show_progress else None
334        self.parser = argparse.ArgumentParser(
335            prog='{0} {1}'.format(getattr(self.parent, 'name', 'UNDEFINED'), self.name),
336            description=self.description)
337        self.add_parser_args(self.parser)
338
339    def __repr__(self):
340        return "Subcommand(name={0})".format(repr(self.name))
341
342    @property
343    def model(self):
344        return self.parent.db_model
345
346    def add_parser_args(self, parser):
347        """
348        Configure additional arguments for the subcommand argument parser.
349        """
350        pass
351
352    def make_session(self):
353        session = self.parent.db_session_factory()
354        user = self.get_runas_user(session=session)
355        if user:
356            session.set_continuum_user(user)
357        return session
358
359    def get_runas_user(self, session=None, username=None):
360        """
361        Returns a proper User object, which the app should "run as"
362        """
363        from sqlalchemy.orm.exc import NoResultFound
364        from rattail.db.util import short_session
365
366        if username is None:
367            if hasattr(self, 'runas_username'):
368                username = self.runas_username
369            if not username and self.config:
370                username = self.config.get('rattail', 'runas.default')
371        if username:
372            user = None
373            with short_session(session) as s:
374                try:
375                    user = s.query(self.model.User).filter_by(username=username).one()
376                except NoResultFound:
377                    pass
378                else:
379                    if not session:
380                        s.expunge(user)
381            return user
382
383    def progress_loop(self, func, items, factory=None, **kwargs):
384        return progress_loop(func, items, factory or self.progress, **kwargs)
385           
386    def _run(self, *args):
387        args = self.parser.parse_args(list(args))
388        return self.run(args)
389
390    def run(self, args):
391        """
392        Run the subcommand logic.
393        """
394        raise NotImplementedError
395
396
397class CloneDatabase(Subcommand):
398    """
399    Clone (supported) data from a source DB to a target DB
400    """
401    name = 'clonedb'
402    description = __doc__.strip()
403
404    def add_parser_args(self, parser):
405        parser.add_argument('source_engine',
406                            help="SQLAlchemy engine URL for the source database.")
407        parser.add_argument('target_engine',
408                            help="SQLAlchemy engine URL for the target database.")
409        parser.add_argument('-m', '--model', default='rattail.db.model',
410                            help="Dotted path of Python module which contains the data model.")
411        parser.add_argument('-C', '--classes', nargs='*',
412                            help="Model classes which should be cloned.  Possible values here "
413                            "depends on which module contains the data model.  If no classes "
414                            "are specified, all available will be cloned.")
415
416    def run(self, args):
417        from sqlalchemy import create_engine, orm
418        from rattail.util import import_module_path
419
420        model = import_module_path(args.model)
421        classes = args.classes
422        assert classes
423
424        source_engine = create_engine(args.source_engine)
425        target_engine = create_engine(args.target_engine)
426        model.Base.metadata.drop_all(bind=target_engine)
427        model.Base.metadata.create_all(bind=target_engine)
428
429        Session = orm.sessionmaker()
430        src_session = Session(bind=source_engine)
431        dst_session = Session(bind=target_engine)
432
433        for clsname in classes:
434            log.info("cloning data for model: %s", clsname)
435            cls = getattr(model, clsname)
436            src_query = src_session.query(cls)
437            count = src_query.count()
438            log.debug("found %d %s records to clone", count, clsname)
439            if not count:
440                continue
441
442            mapper = orm.class_mapper(cls)
443            key_query = src_session.query(*mapper.primary_key)
444
445            prog = None
446            if self.progress:
447                prog = self.progress("Cloning data for model: {0}".format(clsname), count)
448            for i, key in enumerate(key_query, 1):
449
450                src_instance = src_query.get(key)
451                dst_session.merge(src_instance)
452                dst_session.flush()
453
454                if prog:
455                    prog.update(i)
456            if prog:
457                prog.destroy()
458
459        src_session.close()
460        dst_session.commit()
461        dst_session.close()
462
463
464class DataSync(Subcommand):
465    """
466    Manage the data sync daemon
467    """
468    name = 'datasync'
469    description = __doc__.strip()
470
471    def add_parser_args(self, parser):
472        subparsers = parser.add_subparsers(title='subcommands')
473
474        start = subparsers.add_parser('start', help="Start service")
475        start.set_defaults(subcommand='start')
476
477        stop = subparsers.add_parser('stop', help="Stop service")
478        stop.set_defaults(subcommand='stop')
479
480        check = subparsers.add_parser('check', help="(DEPRECATED) Check for stale (lingering) changes in queue")
481        check.set_defaults(subcommand='check')
482
483        check_queue = subparsers.add_parser('check-queue', help="Check for stale (lingering) changes in queue")
484        check_queue.set_defaults(subcommand='check-queue')
485
486        check_watchers = subparsers.add_parser('check-watchers', help="Check for dead watcher threads")
487        check_watchers.set_defaults(subcommand='check-watchers')
488
489        wait = subparsers.add_parser('wait', help="Wait for changes to be processed")
490        wait.set_defaults(subcommand='wait')
491
492        parser.add_argument('-p', '--pidfile', metavar='PATH', default='/var/run/rattail/datasync.pid',
493                            help="Path to PID file.")
494        parser.add_argument('--daemonize', action='store_true', default=True, # TODO: should default to False
495                            help="Daemonize when starting.")
496        parser.add_argument('--no-daemonize',
497                            '-D', '--do-not-daemonize', # TODO: (re)move these?
498                            action='store_false', dest='daemonize',
499                            help="Do not daemonize when starting.")
500        parser.add_argument('-T', '--timeout', metavar='MINUTES', type=int, default=0,
501                            help="Optional timeout (in minutes) for use with the 'wait' or 'check' commands.  "
502                            "If specified for 'wait', the waiting still stop after the given number of minutes "
503                            "and exit with a nonzero code to indicate failure.  If specified for 'check', the "
504                            "command will perform some health check based on the given timeout, and exit with "
505                            "nonzero code if the check fails.")
506
507    def run(self, args):
508        from rattail.datasync.daemon import DataSyncDaemon
509
510        if args.subcommand == 'wait':
511            self.wait(args)
512
513        elif args.subcommand == 'check':
514            self.check_queue(args)
515
516        elif args.subcommand == 'check-queue':
517            self.check_queue(args)
518
519        elif args.subcommand == 'check-watchers':
520            self.check_watchers(args)
521
522        else: # manage the daemon
523            daemon = DataSyncDaemon(args.pidfile, config=self.config)
524            if args.subcommand == 'stop':
525                daemon.stop()
526            else: # start
527                try:
528                    daemon.start(daemonize=args.daemonize)
529                except KeyboardInterrupt:
530                    if not args.daemonize:
531                        self.stderr.write("Interrupted.\n")
532                    else:
533                        raise
534
535    def wait(self, args):
536        model = self.model
537        session = self.make_session()
538        started = make_utc()
539        log.debug("will wait for current change queue to clear")
540        last_logged = started
541
542        changes = session.query(model.DataSyncChange)
543        count = changes.count()
544        log.debug("there are %d changes in the queue", count)
545        while count:
546            try:
547                now = make_utc()
548
549                if args.timeout and (now - started).seconds >= (args.timeout * 60):
550                    log.warning("datasync wait timed out after %d minutes, with %d changes in queue",
551                                args.timeout, count)
552                    sys.exit(1)
553
554                if (now - last_logged).seconds >= 60:
555                    log.debug("still waiting, %d changes in the datasync queue", count)
556                    last_logged = now
557
558                time.sleep(1)
559                count = changes.count()
560
561            except KeyboardInterrupt:
562                self.stderr.write("Waiting cancelled by user\n")
563                session.close()
564                sys.exit(1)
565
566        session.close()
567        log.debug("all datasync changes have been processed")
568
569    def check_queue(self, args):
570        """
571        Perform general queue / health check for datasync.
572        """
573        model = self.model
574        session = self.make_session()
575
576        # looking for changes which have been around for "timeout" minutes
577        timeout = args.timeout or 90
578        cutoff = make_utc() - datetime.timedelta(seconds=60 * timeout)
579        changes = session.query(model.DataSyncChange)\
580                         .filter(model.DataSyncChange.obtained < cutoff)\
581                         .count()
582        session.close()
583
584        # if we found stale changes, then "fail" - otherwise we'll "succeed"
585        if changes:
586            log.debug("found %s changes, in queue for %s minutes", changes, timeout)
587            self.stderr.write("Found {} changes, in queue for {} minutes\n".format(changes, timeout))
588            sys.exit(1)
589
590        log.info("found no changes in queue for %s minutes", timeout)
591
592    def check_watchers(self, args):
593        """
594        Perform general health check for datasync watcher threads.
595        """
596        from rattail.datasync.config import load_profiles
597        from rattail.datasync.util import get_lastrun
598
599        profiles = load_profiles(self.config)
600        session = self.make_session()
601
602        # cutoff is "timeout" minutes before "now"
603        timeout = args.timeout or 15
604        cutoff = make_utc() - datetime.timedelta(seconds=60 * timeout)
605
606        dead = []
607        for key in profiles:
608
609            # looking for watcher "last run" time older than "timeout" minutes
610            lastrun = get_lastrun(self.config, key, tzinfo=False, session=session)
611            if lastrun and lastrun < cutoff:
612                dead.append(key)
613
614        session.close()
615
616        # if we found dead watchers, then "fail" - otherwise we'll "succeed"
617        if dead:
618            self.stderr.write("Found {} watcher threads dead for {} minutes: {}\n".format(len(dead), timeout, ', '.join(dead)))
619            sys.exit(1)
620
621        log.info("found no watcher threads dead for %s minutes", timeout)
622
623
624class EmailBouncer(Subcommand):
625    """
626    Interacts with the email bouncer daemon.  This command expects a
627    subcommand; one of the following:
628
629    * ``rattail bouncer start``
630    * ``rattail bouncer stop``
631    """
632    name = 'bouncer'
633    description = "Manage the email bouncer daemon"
634
635    def add_parser_args(self, parser):
636        subparsers = parser.add_subparsers(title='subcommands')
637
638        start = subparsers.add_parser('start', help="Start service")
639        start.set_defaults(subcommand='start')
640        stop = subparsers.add_parser('stop', help="Stop service")
641        stop.set_defaults(subcommand='stop')
642
643        parser.add_argument('-p', '--pidfile', metavar='PATH', default='/var/run/rattail/bouncer.pid',
644                            help="Path to PID file.")
645        parser.add_argument('--daemonize', action='store_true', default=True, # TODO: should default to False
646                            help="Daemonize when starting.")
647        parser.add_argument('--no-daemonize',
648                            '-D', '--do-not-daemonize', # TODO: (re)move these?
649                            action='store_false', dest='daemonize',
650                            help="Do not daemonize when starting.")
651
652    def run(self, args):
653        from rattail.bouncer.daemon import BouncerDaemon
654
655        daemon = BouncerDaemon(args.pidfile, config=self.config)
656        if args.subcommand == 'stop':
657            daemon.stop()
658        else: # start
659            try:
660                daemon.start(daemonize=args.daemonize)
661            except KeyboardInterrupt:
662                if not args.daemonize:
663                    self.stderr.write("Interrupted.\n")
664                else:
665                    raise
666
667
668class DateOrganize(Subcommand):
669    """
670    Organize files in a given directory, according to date
671    """
672    name = 'date-organize'
673    description = __doc__.strip()
674
675    def add_parser_args(self, parser):
676        parser.add_argument('folder', metavar='PATH',
677                            help="Path to directory containing files which are "
678                            "to be organized by date.")
679
680    def run(self, args):
681        today = datetime.date.today()
682        for filename in sorted(os.listdir(args.folder)):
683            path = os.path.join(args.folder, filename)
684            if os.path.isfile(path):
685                mtime = datetime.datetime.fromtimestamp(os.path.getmtime(path))
686                if mtime.date() < today:
687                    datedir = mtime.strftime(os.sep.join(('%Y', '%m', '%d')))
688                    datedir = os.path.join(args.folder, datedir)
689                    if not os.path.exists(datedir):
690                        os.makedirs(datedir)
691                    shutil.move(path, datedir)
692
693
694class DatabaseSyncCommand(Subcommand):
695    """
696    Controls the database synchronization service.
697    """
698
699    name = 'dbsync'
700    description = "Manage the database synchronization service"
701
702    def add_parser_args(self, parser):
703        subparsers = parser.add_subparsers(title='subcommands')
704
705        start = subparsers.add_parser('start', help="Start service")
706        start.set_defaults(subcommand='start')
707        stop = subparsers.add_parser('stop', help="Stop service")
708        stop.set_defaults(subcommand='stop')
709
710        if sys.platform == 'linux2':
711            parser.add_argument('-p', '--pidfile',
712                                help="Path to PID file", metavar='PATH')
713            parser.add_argument('-D', '--do-not-daemonize',
714                                action='store_false', dest='daemonize', default=True,
715                                help="Do not daemonize when starting.")
716
717    def run(self, args):
718        from rattail.db.sync import linux as dbsync
719
720        if args.subcommand == 'start':
721            try:
722                dbsync.start_daemon(self.config, args.pidfile, args.daemonize)
723            except KeyboardInterrupt:
724                if not args.daemonize:
725                    self.stderr.write("Interrupted.\n")
726                else:
727                    raise
728
729        elif args.subcommand == 'stop':
730            dbsync.stop_daemon(self.config, args.pidfile)
731
732
733class Dump(Subcommand):
734    """
735    Do a simple data dump.
736    """
737
738    name = 'dump'
739    description = "Dump data to file."
740
741    def add_parser_args(self, parser):
742        parser.add_argument(
743            '--output', '-o', metavar='FILE',
744            help="Optional path to output file.  If none is specified, "
745            "data will be written to standard output.")
746        parser.add_argument(
747            'model', help="Model whose data will be dumped.")
748
749    def get_model(self):
750        """
751        Returns the module which contains all relevant data models.
752
753        By default this returns ``rattail.db.model``, but this method may be
754        overridden in derived commands to add support for extra data models.
755        """
756        from rattail.db import model
757        return model
758
759    def run(self, args):
760        from rattail.db import Session
761        from rattail.db.dump import dump_data
762
763        model = self.get_model()
764        if hasattr(model, args.model):
765            cls = getattr(model, args.model)
766        else:
767            self.stderr.write("Unknown model: {0}\n".format(args.model))
768            sys.exit(1)
769
770        progress = None
771        if self.show_progress: # pragma no cover
772            progress = ConsoleProgress
773
774        if args.output:
775            output = open(args.output, 'wb')
776        else:
777            output = self.stdout
778
779        session = Session()
780        dump_data(session, cls, output, progress=progress)
781        session.close()
782
783        if output is not self.stdout:
784            output.close()
785
786
787class FileMonitorCommand(Subcommand):
788    """
789    Interacts with the file monitor service; called as ``rattail filemon``.
790    This command expects a subcommand; one of the following:
791
792    * ``rattail filemon start``
793    * ``rattail filemon stop``
794
795    On Windows platforms, the following additional subcommands are available:
796
797    * ``rattail filemon install``
798    * ``rattail filemon uninstall`` (or ``rattail filemon remove``)
799
800    .. note::
801       The Windows Vista family of operating systems requires you to launch
802       ``cmd.exe`` as an Administrator in order to have sufficient rights to
803       run the above commands.
804
805    .. See :doc:`howto.use_filemon` for more information.
806    """
807
808    name = 'filemon'
809    description = "Manage the file monitor daemon"
810
811    def add_parser_args(self, parser):
812        subparsers = parser.add_subparsers(title='subcommands')
813
814        start = subparsers.add_parser('start', help="Start service")
815        start.set_defaults(subcommand='start')
816        stop = subparsers.add_parser('stop', help="Stop service")
817        stop.set_defaults(subcommand='stop')
818
819        if sys.platform in ('linux', 'linux2'):
820            parser.add_argument('-p', '--pidfile',
821                                help="Path to PID file.", metavar='PATH')
822            parser.add_argument('--daemonize', action='store_true', default=True, # TODO: should default to False
823                                help="Daemonize when starting.")
824            parser.add_argument('--no-daemonize',
825                                '-D', '--do-not-daemonize', # TODO: (re)move these?
826                                action='store_false', dest='daemonize',
827                                help="Do not daemonize when starting.")
828
829        elif sys.platform == 'win32': # pragma no cover
830
831            install = subparsers.add_parser('install', help="Install service")
832            install.set_defaults(subcommand='install')
833            install.add_argument('-a', '--auto-start', action='store_true',
834                                 help="Configure service to start automatically.")
835            install.add_argument('-U', '--username',
836                                 help="User account under which the service should run.")
837
838            remove = subparsers.add_parser('remove', help="Uninstall (remove) service")
839            remove.set_defaults(subcommand='remove')
840
841            uninstall = subparsers.add_parser('uninstall', help="Uninstall (remove) service")
842            uninstall.set_defaults(subcommand='remove')
843
844    def run(self, args):
845        if sys.platform in ('linux', 'linux2'):
846            from rattail.filemon import linux as filemon
847
848            if args.subcommand == 'start':
849                filemon.start_daemon(self.config, args.pidfile, args.daemonize)
850
851            elif args.subcommand == 'stop':
852                filemon.stop_daemon(self.config, args.pidfile)
853
854        elif sys.platform == 'win32': # pragma no cover
855            self.run_win32(args)
856
857        else:
858            self.stderr.write("File monitor is not supported on platform: {0}\n".format(sys.platform))
859            sys.exit(1)
860
861    def run_win32(self, args): # pragma no cover
862        from rattail.win32 import require_elevation
863        from rattail.win32 import service
864        from rattail.win32 import users
865        from rattail.filemon import win32 as filemon
866
867        require_elevation()
868
869        options = []
870        if args.subcommand == 'install':
871
872            username = args.username
873            if username:
874                if '\\' in username:
875                    server, username = username.split('\\')
876                else:
877                    server = socket.gethostname()
878                if not users.user_exists(username, server):
879                    sys.stderr.write("User does not exist: {0}\\{1}\n".format(server, username))
880                    sys.exit(1)
881
882                password = ''
883                while password == '':
884                    password = getpass(b"Password for service user: ").strip()
885                options.extend(['--username', r'{0}\{1}'.format(server, username)])
886                options.extend(['--password', password])
887
888            if args.auto_start:
889                options.extend(['--startup', 'auto'])
890
891        service.execute_service_command(filemon, args.subcommand, *options)
892
893        # If installing with custom user, grant "logon as service" right.
894        if args.subcommand == 'install' and args.username:
895            users.allow_logon_as_service(username)
896
897        # TODO: Figure out if the following is even required, or if instead we
898        # should just be passing '--startup delayed' to begin with?
899
900        # If installing auto-start service on Windows 7, we should update
901        # its startup type to be "Automatic (Delayed Start)".
902        # TODO: Improve this check to include Vista?
903        if args.subcommand == 'install' and args.auto_start:
904            if platform.release() == '7':
905                name = filemon.RattailFileMonitor._svc_name_
906                service.delayed_auto_start_service(name)
907
908
909class LoadHostDataCommand(Subcommand):
910    """
911    Loads data from the Rattail host database, if one is configured.
912    """
913
914    name = 'load-host-data'
915    description = "Load data from host database"
916
917    def run(self, args):
918        from .db import get_engines
919        from .db import load
920
921        engines = get_engines(self.config)
922        if 'host' not in engines:
923            sys.stderr.write("Host engine URL not configured.\n")
924            sys.exit(1)
925
926        proc = load.LoadProcessor(self.config)
927        proc.load_all_data(engines['host'], ConsoleProgress)
928
929
930class MakeAppDir(Subcommand):
931    """
932    Create a conventional 'app' dir for a virtual environment
933    """
934    name = 'make-appdir'
935    description = __doc__.strip()
936
937    def add_parser_args(self, parser):
938        parser.add_argument('--path', metavar='PATH',
939                            help="Path where the app folder is to be established.  If not "
940                            "specified, a default will be assumed based on the virtual "
941                            "environment, e.g. '/envs/rattail/app'.")
942        parser.add_argument('-U', '--user', metavar='USERNAME',
943                            help="Linux username which should be given ownership to the various "
944                            "data folders which are to be created.  This is used when the app(s) "
945                            "are to normally be ran as the 'rattail' user for instance.  Use "
946                            "of this option probably requires 'sudo' or equivalent.")
947
948    def run(self, args):
949        import pwd
950
951        if args.path:
952            app_path = os.path.abspath(args.path)
953            if os.path.basename(app_path) != 'app':
954                app_path = os.path.join(app_path, 'app')
955        else:
956            app_path = os.path.join(sys.prefix, 'app')
957        app_path = app_path.rstrip('/')
958
959        if not os.path.exists(app_path):
960            os.mkdir(app_path)
961
962        if args.user:
963            pwdata = pwd.getpwnam(args.user)
964        for name in ['batch', 'log', 'sessions', 'work']:
965            path = os.path.join(app_path, name)
966            if not os.path.exists(path):
967                os.mkdir(path)
968            if args.user:
969                os.chown(path, pwdata.pw_uid, pwdata.pw_gid)
970
971        self.stdout.write("Rattail app dir generated at: {}\n".format(app_path))
972
973
974class MakeConfig(Subcommand):
975    """
976    Generate stub config file(s) where you want them
977    """
978    name = 'make-config'
979    description = __doc__.strip()
980
981    def add_parser_args(self, parser):
982        parser.add_argument('-T', '--type', metavar='NAME', default='rattail',
983                            help="Type of config file to create; defaults to 'rattail' "
984                            "which will generate 'rattail.conf'")
985        parser.add_argument('-O', '--output', metavar='PATH', default='.',
986                            help="Path where the config file is to be generated.  This can "
987                            "be the full path including filename, or just the folder, in which "
988                            "case the filename is inferred from 'type'.  Default is to current "
989                            "working folder.")
990
991    def find_template(self, name):
992        from rattail.files import resource_path
993
994        template_paths = self.config.getlist('rattail.config', 'templates',
995                                             default=['rattail:data/config'])
996        for template_path in template_paths:
997            path = resource_path('{}/{}.conf'.format(template_path.rstrip('/'), name))
998            if os.path.exists(path):
999                return path
1000
1001    def run(self, args):
1002        template_path = self.find_template(args.type)
1003        if not template_path:
1004            self.stderr.write("config template not found for type: {}\n".format(args.type))
1005            sys.exit(1)
1006
1007        output_path = os.path.abspath(args.output)
1008        if os.path.isdir(output_path):
1009            output_path = os.path.join(output_path, os.path.basename(template_path))
1010
1011        shutil.copy(template_path, output_path)
1012        self.stdout.write("Config file generated at: {}\n".format(output_path))
1013
1014
1015class MakeUser(Subcommand):
1016    """
1017    Create a new user account in a given system
1018    """
1019    name = 'make-user'
1020    description = __doc__.strip()
1021
1022    def add_parser_args(self, parser):
1023        parser.add_argument('username',
1024                            help="Username for the new user.")
1025        parser.add_argument('--system', default='rattail',
1026                            help="System in which to create the new user; defaults to "
1027                            "rattail; must be one of: rattail, windows")
1028        parser.add_argument('-A', '--admin', action='store_true',
1029                            help="Whether the new user should have admin rights within "
1030                            "the system (if applicable).")
1031        parser.add_argument('--password',
1032                            help="Optional password to set for the new user.")
1033        parser.add_argument('--full-name',
1034                            help="Full (display) name for the new user (if applicable).")
1035        parser.add_argument('--comment',
1036                            help="Comment string for the new user (if applicable).")
1037
1038    def run(self, args):
1039        mkuser = getattr(self, 'mkuser_{}'.format(args.system), None)
1040        if mkuser:
1041            if mkuser(args):
1042                self.stdout.write("created new user in '{}' system: {}\n".format(
1043                    args.system, args.username))
1044        else:
1045            self.stderr.write("don't know how to make user for '{}' system\n".format(args.system))
1046            sys.exit(1)
1047
1048    def user_exists(self, args):
1049        self.stdout.write("user already exists in '{}' system: {}\n".format(
1050            args.system, args.username))
1051        sys.exit(1)
1052
1053    def obtain_password(self, args):
1054        if args.password:
1055            return args.password
1056        try:
1057            password = None
1058            while not password:
1059                password = getpass(b"enter password for new user: ").strip()
1060        except KeyboardInterrupt:
1061            self.stderr.write("\noperation canceled by user\n")
1062            sys.exit(2)
1063        return password
1064
1065    def mkuser_rattail(self, args):
1066        from rattail.db import auth
1067
1068        session = self.parent.db_session_factory()
1069        model = self.parent.db_model
1070        if session.query(model.User).filter_by(username=args.username).count():
1071            session.close()
1072            return self.user_exists(args)
1073
1074        user = model.User(username=args.username)
1075        auth.set_user_password(user, self.obtain_password(args))
1076        if args.admin:
1077            user.roles.append(auth.administrator_role(session))
1078        if args.full_name:
1079            kwargs = {'display_name': args.full_name}
1080            words = args.full_name.split()
1081            if len(words) == 2:
1082                kwargs.update({'first_name': words[0], 'last_name': words[1]})
1083            user.person = model.Person(**kwargs)
1084        session.add(user)
1085        session.commit()
1086        session.close()
1087        return True
1088
1089    def mkuser_windows(self, args):
1090        if sys.platform != 'win32':
1091            self.stderr.write("sorry, only win32 platform is supported\n")
1092            sys.exit(1)
1093
1094        from rattail.win32 import users
1095        from rattail.win32 import require_elevation
1096
1097        require_elevation()
1098
1099        if users.user_exists(args.username):
1100            return self.user_exists(args)
1101
1102        return users.create_user(args.username, self.obtain_password(args),
1103                                 full_name=args.full_name, comment=args.comment)
1104
1105
1106class MakeUUID(Subcommand):
1107    """
1108    Generate a new UUID
1109    """
1110    name = 'make-uuid'
1111    description = __doc__.strip()
1112
1113    def run(self, args):
1114        from rattail.core import get_uuid
1115
1116        self.stdout.write("{}\n".format(get_uuid()))
1117
1118
1119class PalmCommand(Subcommand):
1120    """
1121    Manages registration for the HotSync Manager conduit; called as::
1122
1123       rattail palm
1124    """
1125
1126    name = 'palm'
1127    description = "Manage the HotSync Manager conduit registration"
1128
1129    def add_parser_args(self, parser):
1130        subparsers = parser.add_subparsers(title='subcommands')
1131
1132        register = subparsers.add_parser('register', help="Register Rattail conduit")
1133        register.set_defaults(subcommand='register')
1134
1135        unregister = subparsers.add_parser('unregister', help="Unregister Rattail conduit")
1136        unregister.set_defaults(subcommand='unregister')
1137
1138    def run(self, args):
1139        from rattail import palm
1140        from rattail.win32 import require_elevation
1141        from rattail.exceptions import PalmError
1142
1143        require_elevation()
1144
1145        if args.subcommand == 'register':
1146            try:
1147                palm.register_conduit()
1148            except PalmError as error:
1149                sys.stderr.write("{}\n".format(error))
1150
1151        elif args.subcommand == 'unregister':
1152            try:
1153                palm.unregister_conduit()
1154            except PalmError as error:
1155                sys.stderr.write("{}\n".format(error))
1156               
1157
1158class RunAndMail(Subcommand):
1159    """
1160    Run a command as subprocess, and email the result/output
1161    """
1162    name = 'run-n-mail'
1163    description = __doc__.strip()
1164
1165    def add_parser_args(self, parser):
1166        parser.add_argument('--key', default='run_n_mail',
1167                            help="Config key for email settings")
1168        # TODO: these all seem like good ideas, but not needed yet?
1169        # parser.add_argument('--from', '-F', metavar='ADDRESS',
1170        #                     help="Override value of From: header")
1171        # parser.add_argument('--to', '-T', metavar='ADDRESS',
1172        #                     help="Override value of To: header (may specify more than once)")
1173        # parser.add_argument('--cc', metavar='ADDRESS',
1174        #                     help="Override value of Cc: header (may specify more than once)")
1175        # parser.add_argument('--bcc', metavar='ADDRESS',
1176        #                     help="Override value of Bcc: header (may specify more than once)")
1177        parser.add_argument('--subject', '-S',
1178                            help="Override value of Subject: header (i.e. value after prefix)")
1179        parser.add_argument('cmd', metavar='COMMAND',
1180                            help="Command which should be ran, and result of which will be emailed")
1181
1182    def run(self, args):
1183        from rattail.mail import send_email
1184
1185        cmd = parse_list(args.cmd)
1186        log.info("will run command as subprocess: %s", cmd)
1187        run_began = make_utc()
1188
1189        try:
1190            # TODO: must we allow for shell=True in some situations? (clearly not yet)
1191            output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
1192            retcode = 0
1193            log.info("command completed successfully")
1194        except subprocess.CalledProcessError as error:
1195            output = error.output
1196            retcode = error.returncode
1197            log.warn("command exited with code: %s", retcode)
1198
1199        run_ended = make_utc()
1200        runtime = run_ended - run_began
1201        runtime_pretty = humanize.naturaldelta(runtime)
1202
1203        kwargs = {}
1204        if args.subject:
1205            kwargs['subject_template'] = args.subject
1206
1207        send_email(self.config, args.key, {
1208            'cmd': cmd,
1209            'run_began': run_began,
1210            'run_ended': run_ended,
1211            'runtime': runtime,
1212            'runtime_pretty': runtime_pretty,
1213            'retcode': retcode,
1214            'output': output,
1215        }, **kwargs)
1216
1217
1218class RunSQL(Subcommand):
1219    """
1220    Run (first statement of) a SQL script against a database
1221    """
1222    name = 'runsql'
1223    description = __doc__.strip()
1224
1225    def add_parser_args(self, parser):
1226        parser.add_argument('engine',
1227                            help="SQLAlchemy engine URL for the database.")
1228        parser.add_argument('script', type=argparse.FileType('r'),
1229                            help="Path to file which contains a SQL script.")
1230        parser.add_argument('--max-width', type=int, default=80,
1231                            help="Max table width when displaying results.")
1232
1233    def run(self, args):
1234        import sqlalchemy as sa
1235        import texttable
1236
1237        sql = []
1238        for line in args.script:
1239            line = line.strip()
1240            if line and not line.startswith('--'):
1241                sql.append(line)
1242                if line.endswith(';'):
1243                    break
1244
1245        sql = ' '.join(sql)
1246        engine = sa.create_engine(args.engine)
1247
1248        result = engine.execute(sql)
1249        rows = result.fetchall()
1250        if rows:
1251            table = texttable.Texttable(max_width=args.max_width)
1252            table.add_rows([rows[0].keys()] + rows)
1253            self.stdout.write("{}\n".format(table.draw()))
1254
1255
1256class Upgrade(Subcommand):
1257    """
1258    Upgrade the local Rattail app
1259    """
1260    name = 'upgrade'
1261    description = __doc__.strip()
1262
1263    def add_parser_args(self, parser):
1264        parser.add_argument('--description',
1265                            help="Description for the new/matched upgrade.")
1266        parser.add_argument('--enabled', action='store_true', default=True,
1267                            help="Indicate the enabled flag should be ON for the new/matched upgrade.  "
1268                            "Note that this is the default if you do not specify.")
1269        parser.add_argument('--no-enabled', action='store_false', dest='enabled',
1270                            help="Indicate the enabled flag should be OFF for the new/matched upgrade.")
1271        parser.add_argument('--create', action='store_true',
1272                            help="Create a new upgrade with the given attributes.")
1273        parser.add_argument('--execute', action='store_true',
1274                            help="Execute the upgrade.  Note that if you do not specify "
1275                            "--create then the upgrade matching the given attributes "
1276                            "will be read from the database.  If such an upgrade is not "
1277                            "found or is otherwise invalid (e.g. already executed), "
1278                            "the command will fail.")
1279        parser.add_argument('--dry-run', action='store_true',
1280                            help="Go through the full motions and allow logging etc. to "
1281                            "occur, but rollback (abort) the transaction at the end.")
1282
1283    def run(self, args):
1284        from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
1285        from rattail.upgrades import get_upgrade_handler
1286
1287        if not args.create and not args.execute:
1288            self.stderr.write("Must specify --create and/or --execute\n")
1289            sys.exit(1)
1290
1291        session = self.make_session()
1292        model = self.model
1293        user = self.get_runas_user(session)
1294
1295        if args.create:
1296            upgrade = model.Upgrade()
1297            upgrade.description = args.description
1298            upgrade.created = make_utc()
1299            upgrade.created_by = user
1300            upgrade.enabled = args.enabled
1301            session.add(upgrade)
1302            session.flush()
1303            log.info("user '%s' created new upgrade: %s", user.username, upgrade)
1304
1305        else:
1306            upgrades = session.query(model.Upgrade)\
1307                              .filter(model.Upgrade.enabled == args.enabled)
1308            if args.description:
1309                upgrades = upgrades.filter(model.Upgrade.description == args.description)
1310            try:
1311                upgrade = upgrades.one()
1312            except NoResultFound:
1313                self.stderr.write("no matching upgrade found\n")
1314                session.rollback()
1315                session.close()
1316                sys.exit(1)
1317            except MultipleResultsFound:
1318                self.stderr.write("found {} matching upgrades\n".format(upgrades.count()))
1319                session.rollback()
1320                session.close()
1321                sys.exit(1)
1322
1323        if args.execute:
1324            if upgrade.executed:
1325                self.stderr.write("upgrade has already been executed: {}\n".format(upgrade))
1326                session.rollback()
1327                session.close()
1328                sys.exit(1)
1329            if not upgrade.enabled:
1330                self.stderr.write("upgrade is not enabled for execution: {}\n".format(upgrade))
1331                session.rollback()
1332                session.close()
1333                sys.exit(1)
1334
1335            # execute upgrade
1336            handler = get_upgrade_handler(self.config)
1337            log.info("will now execute upgrade: %s", upgrade)
1338            if not args.dry_run:
1339                handler.mark_executing(upgrade)
1340                session.commit()
1341                handler.do_execute(upgrade, user, progress=self.progress)
1342            log.info("user '%s' executed upgrade: %s", user.username, upgrade)
1343
1344        if args.dry_run:
1345            session.rollback()
1346            log.info("dry run, so transaction was rolled back")
1347        else:
1348            session.commit()
1349            log.info("transaction was committed")
1350        session.close()
1351
1352
1353def main(*args):
1354    """
1355    The primary entry point for the Rattail command system.
1356    """
1357    if args:
1358        args = list(args)
1359    else:
1360        args = sys.argv[1:]
1361
1362    cmd = Command()
1363    cmd.run(*args)
Note: See TracBrowser for help on using the repository browser.