source: rattail/rattail/batch/handlers.py @ 9df0f00

Last change on this file since 9df0f00 was 9df0f00, checked in by Lance Edgar <ledgar@…>, 18 months ago

Don't raise error if "removing" a batch row which was already "removed"

just let caller assume the row was removed okay

  • Property mode set to 100644
File size: 24.9 KB
Line 
1# -*- coding: utf-8; -*-
2################################################################################
3#
4#  Rattail -- Retail Software Framework
5#  Copyright © 2010-2018 Lance Edgar
6#
7#  This file is part of Rattail.
8#
9#  Rattail is free software: you can redistribute it and/or modify it under the
10#  terms of the GNU General Public License as published by the Free Software
11#  Foundation, either version 3 of the License, or (at your option) any later
12#  version.
13#
14#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
15#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
16#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
17#  details.
18#
19#  You should have received a copy of the GNU General Public License along with
20#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
21#
22################################################################################
23"""
24Data Batch Handlers
25"""
26
27from __future__ import unicode_literals, absolute_import
28
29import os
30import shutil
31import datetime
32import warnings
33
34from sqlalchemy import orm
35
36from rattail.core import Object
37from rattail.db.cache import cache_model
38from rattail.time import localtime, make_utc
39from rattail.util import progress_loop, load_object
40
41
42class BatchHandler(object):
43    """
44    Base class and partial default implementation for batch handlers.  It is
45    expected that all batch handlers will ultimately inherit from this base
46    class, therefore it defines the implementation "interface" loosely
47    speaking.  Custom batch handlers are welcome to supplement or override this
48    as needed, and in fact must do so for certain aspects.
49
50    .. attribute:: populate_batches
51
52       Simple flag to indicate whether any/all batches being handled, will
53       require initial population from a relevant data source.  Note that this
54       flag should be set to ``True`` if *any* batches may need population.
55       Whether or not a given batch actually needs to be populated, is
56       ultimately determined by the :meth:`should_populate()` method.
57
58    .. attribute:: populate_with_versioning
59
60       This flag indicates whether it's okay for data versioning to be enabled
61       during initial batch population.
62
63       If set to ``True`` (the default), then versioning is allowed and
64       therefore the caller need take no special precautions when populating
65       the batch.
66
67       If set to ``False`` then versioning is *not* allowed; if versioning is
68       not enabled for the current process, the caller may populate the batch
69       with no special precautions.  However if versioning *is* enabled, the
70       caller must launch a separate process with versioning disabled, in order
71       to populate the batch.
72
73    .. attribute:: refresh_with_versioning
74
75       This flag indicates whether it's okay for data versioning to be enabled
76       during batch refresh.
77
78       If set to ``True`` (the default), then versioning is allowed and
79       therefore the caller need take no special precautions when populating
80       the batch.
81
82       If set to ``False`` then versioning is *not* allowed; if versioning is
83       not enabled for the current process, the caller may populate the batch
84       with no special precautions.  However if versioning *is* enabled, the
85       caller must launch a separate process with versioning disabled, in order
86       to refresh the batch.
87
88    .. attribute:: execute_with_versioning
89
90       This flag indicates whether it's okay for data versioning to be enabled
91       during batch execution.
92
93       If set to ``True`` (the default), then versioning is allowed and
94       therefore the caller need take no special precautions when populating
95       the batch.
96
97       If set to ``False`` then versioning is *not* allowed; if versioning is
98       not enabled for the current process, the caller may populate the batch
99       with no special precautions.  However if versioning *is* enabled, the
100       caller must launch a separate process with versioning disabled, in order
101       to execute the batch.
102
103    .. attribute:: repopulate_when_refresh
104
105       Flag to indicate that when a batch is refreshed, the first step of that
106       should be to re-populate the batch.  The flag is ``False`` by default,
107       in which case the batch is *not* repopulated, i.e. the refresh will work
108       with existing batch rows.
109    """
110    populate_batches = False
111    populate_with_versioning = True
112
113    refresh_with_versioning = True
114    repopulate_when_refresh = False
115
116    execute_with_versioning = True
117
118    def __init__(self, config):
119        self.config = config
120        self.enum = config.get_enum()
121
122    @property
123    def batch_model_class(self):
124        """
125        Reference to the data model class of the batch type for which this
126        handler is responsible, e.g. :class:`rattail.db.model.LabelBatch`.
127        Each handler must define this (or inherit from one that does).
128        """
129        raise NotImplementedError("You must set the 'batch_model_class' attribute "
130                                  "for class '{}'".format(self.__class__.__name__))
131
132    @property
133    def batch_key(self):
134        """
135        The "batch type key" for the handler, e.g. ``'labels'``.  This isn't
136        necessarily unique among handlers, but instead refers to a unique key
137        for the type of batch being handled.  The handler needn't define this,
138        as it is borrowed from :attr:`batch_model_class`.
139        """
140        return self.batch_model_class.batch_key
141
142    def get_model_title(self):
143        return self.batch_model_class.get_model_title()
144
145    def allow_versioning(self, action):
146        if action == 'populate':
147            return self.populate_with_versioning
148        if action == 'refresh':
149            return self.refresh_with_versioning
150        if action == 'execute':
151            return self.execute_with_versioning
152        raise NotImplementedError("unknown batch action: {}".format(action))
153
154    def make_basic_batch(self, session, progress=None, **kwargs):
155        """
156        Make a new "basic" batch, with no customization beyond what is provided
157        by ``kwargs``, which are passed directly to the batch class constructor.
158        """
159        kwargs.setdefault('rowcount', 0)
160        kwargs.setdefault('complete', False)
161        batch = self.batch_model_class(**kwargs)
162        session.add(batch)
163        session.flush()
164        return batch
165
166    def make_batch(self, session, progress=None, **kwargs):
167        """
168        Make a new batch, with initial rows if applicable.
169        """
170        batch = self.make_basic_batch(session, progress=progress, **kwargs)
171        self.init_batch(batch, progress=progress, **kwargs)
172        return batch
173
174    def init_batch(self, batch, progress=None, **kwargs):
175        """
176        Initialize the batch in whatever way might make sense.  Whether this is
177        required at all is up to the batch handler etc.
178        """
179
180    def add_row(self, batch, row):
181        """
182        (Try to) Add the given row to the given batch.  This assumes it is a
183        *new* row, perhaps along with other assumptions?
184        """
185        session = orm.object_session(batch)
186        with session.no_autoflush:
187            batch.data_rows.append(row)
188            self.refresh_row(row)
189        if not row.removed:
190            batch.rowcount += 1
191
192    def purge_batches(self, session, before=None, before_days=90,
193                      delete_all_data=True, progress=None, **kwargs):
194        """
195        Purge all batches which were executed prior to a given date.
196
197        :param before: If provided, must be a timezone-aware datetime object.
198           If not provided, it will be calculated from the current date, using
199           ``before_days``.
200
201        :param before_days: Number of days before the current date, to be used
202           as the cutoff date if ``before`` is not specified.
203
204        :param delete_all_data: Flag indicating whether *all* data should be
205           deleted for each batch being purged.  This flag is passed along to
206           :meth:`delete()`; see that for more info.
207
208        :returns: Integer indicating the number of batches purged.
209        """
210        if not before:
211            before = localtime(self.config).date() - datetime.timedelta(days=before_days)
212            before = datetime.datetime.combine(before, datetime.time(0))
213            before = localtime(self.config, before)
214
215        old_batches = session.query(self.batch_model_class)\
216                             .filter(self.batch_model_class.executed < before)\
217                             .options(orm.joinedload(self.batch_model_class.data_rows))
218        result = Object()
219        result.purged = 0
220
221        def purge(batch, i):
222            self.delete(batch, delete_all_data=delete_all_data, progress=progress)
223            session.delete(batch)
224            result.purged += 1
225            if i % 5 == 0:
226                session.flush()
227
228        self.progress_loop(purge, old_batches, progress,
229                           message="Purging old batches")
230
231        session.flush()
232        return result.purged
233
234    @property
235    def root_datadir(self):
236        """
237        The absolute path of the root folder in which data for this particular
238        type of batch is stored.  The structure of this path is as follows:
239
240        .. code-block:: none
241
242           /{root_batch_data_dir}/{batch_type_key}
243
244        * ``{root_batch_data_dir}`` - Value of the 'batch.files' option in the
245          [rattail] section of config file.
246        * ``{batch_type_key}`` - Unique key for the type of batch it is.
247
248        .. note::
249           While it is likely that the data folder returned by this method
250           already exists, this method does not guarantee it.
251        """
252        return self.config.batch_filedir(self.batch_key)
253
254    def datadir(self, batch):
255        """
256        Returns the absolute path of the folder in which the batch's source
257        data file(s) resides.  Note that the batch must already have been
258        persisted to the database.  The structure of the path returned is as
259        follows:
260
261        .. code-block:: none
262
263           /{root_datadir}/{uuid[:2]}/{uuid[2:]}
264
265        * ``{root_datadir}`` - Value returned by :meth:`root_datadir()`.
266        * ``{uuid[:2]}`` - First two characters of batch UUID.
267        * ``{uuid[2:]}`` - All batch UUID characters *after* the first two.
268
269        .. note::
270           While it is likely that the data folder returned by this method
271           already exists, this method does not guarantee any such thing.  It
272           is typically assumed that the path will have been created by a
273           previous call to :meth:`make_batch()` however.
274        """
275        return os.path.join(self.root_datadir, batch.uuid[:2], batch.uuid[2:])
276
277    def make_datadir(self, batch):
278        """
279        Returns the data folder specific to the given batch, creating if necessary.
280        """
281        datadir = self.datadir(batch)
282        os.makedirs(datadir)
283        return datadir
284
285    # TODO: remove default attr?
286    def set_input_file(self, batch, path, attr='filename'):
287        """
288        Assign the data file found at ``path`` to the batch.  This overwrites
289        the given attribute (``attr``) of the batch and places a copy of the
290        data file in the batch's data folder.
291        """
292        datadir = self.make_datadir(batch)
293        filename = os.path.basename(path)
294        shutil.copyfile(path, os.path.join(datadir, filename))
295        setattr(batch, attr, filename)
296
297    def should_populate(self, batch):
298        """
299        Must return a boolean indicating whether the given batch should be
300        populated from an initial data source, i.e. at time of batch creation.
301        Override this method if you need to inspect the batch in order to
302        determine whether the populate step is needed.  Default behavior is to
303        simply return the value of :attr:`populate_batches`.
304        """
305        return self.populate_batches
306
307    def setup_populate(self, batch, progress=None):
308        """
309        Perform any setup (caching etc.) necessary for populating a batch.
310        """
311
312    def teardown_populate(self, batch, progress=None):
313        """
314        Perform any teardown (cleanup etc.) necessary after populating a batch.
315        """
316
317    def do_populate(self, batch, user, progress=None):
318        """
319        Perform initial population for the batch, i.e. fill it with data rows.
320        Where the handler obtains the data to do this, will vary greatly.
321
322        Note that callers *should* use this method, but custom batch handlers
323        should *not* override this method.  Conversely, custom handlers
324        *should* override the :meth:`~populate()` method, but callers should
325        *not* use that one directly.
326        """
327        self.setup_populate(batch, progress=progress)
328        self.populate(batch, progress=progress)
329        self.teardown_populate(batch, progress=progress)
330        self.refresh_batch_status(batch)
331        return True
332
333    def populate(self, batch, progress=None):
334        """
335        Populate the batch with initial data rows.  It is assumed that the data
336        source to be used will be known by inspecting various properties of the
337        batch itself.
338
339        Note that callers should *not* use this method, but custom batch
340        handlers *should* override this method.  Conversely, custom handlers
341        should *not* override the :meth:`~do_populate()` method, but callers
342        *should* use that one directly.
343        """
344        raise NotImplementedError("Please implement `{}.populate()` method".format(batch.__class__.__name__))
345
346    def refreshable(self, batch):
347        """
348        This method should return a boolean indicating whether or not the
349        handler supports a "refresh" operation for the batch, given its current
350        condition.  The default assumes a refresh is allowed unless the batch
351        is executed.
352
353        Note that this (currently) only affects the enabled/disabled state of
354        the Refresh button within the Tailbone batch view.
355        """
356        if batch.executed:
357            return False
358        return True
359
360    def progress_loop(self, *args, **kwargs):
361        return progress_loop(*args, **kwargs)
362
363    def setup_refresh(self, batch, progress=None):
364        """
365        Perform any setup (caching etc.) necessary for refreshing a batch.
366        """
367
368    def teardown_refresh(self, batch, progress=None):
369        """
370        Perform any teardown (cleanup etc.) necessary after refreshing a batch.
371        """
372
373    def do_refresh(self, batch, user, progress=None):
374        self.refresh(batch, progress=progress)
375        return True
376
377    def refresh(self, batch, progress=None):
378        """
379        Perform a full data refresh for the batch.  What exactly this means will
380        depend on the type of batch, and specific handler logic.
381
382        Generally speaking this refresh is meant to use queries etc. to obtain
383        "fresh" data for the batch (header) and all its rows.  In most cases
384        certain data is expected to be "core" to the batch and/or rows, and
385        such data will be left intact, with all *other* data values being
386        re-calculated and/or reset etc.
387        """
388        session = orm.object_session(batch)
389        self.setup_refresh(batch, progress=progress)
390        if self.repopulate_when_refresh:
391            del batch.data_rows[:]
392            batch.rowcount = 0
393            session.flush()
394            self.populate(batch, progress=progress)
395        else:
396            batch.rowcount = 0
397
398            def refresh(row, i):
399                with session.no_autoflush:
400                    self.refresh_row(row)
401                if not row.removed:
402                    batch.rowcount += 1
403
404            self.progress_loop(refresh, batch.active_rows(), progress,
405                               message="Refreshing batch data rows")
406        self.refresh_batch_status(batch)
407        self.teardown_refresh(batch, progress=progress)
408        return True
409
410    def refresh_row(self, row):
411        """
412        This method will be passed a row object which has already been properly
413        added to a batch, and which has basic required fields already
414        populated.  This method is then responsible for further populating all
415        applicable fields for the row, based on current data within the
416        relevant system(s).
417
418        Note that in some cases this method may be called multiple times for
419        the same row, e.g. once when first creating the batch and then later
420        when a user explicitly refreshes the batch.  The method logic must
421        account for this possibility.
422        """
423
424    def remove_row(self, row):
425        """
426        Remove the given row from its batch.  This may delete the row outright
427        from the database, or simply mark it as removed etc.  Defaults to the
428        latter.
429        """
430        if row.removed:
431            return
432        batch = row.batch
433        row.removed = True
434        self.refresh_batch_status(batch)
435        if batch.rowcount is not None:
436            batch.rowcount -= 1
437
438    def refresh_batch_status(self, batch):
439        """
440        Update the batch status, as needed...
441        """
442
443    def mark_complete(self, batch, progress=None):
444        """
445        Mark the given batch as "complete".  This usually is just a matter of
446        setting the :attr:`~rattail.db.model.batch.BatchMixin.complete` flag
447        for the batch, with the idea that this should "freeze" the batch so
448        that another user can verify its state before finally executing it.
449
450        Each handler is of course free to expound on this idea, or to add extra
451        logic to this "event" of marking a batch complete.
452        """
453        batch.complete = True
454
455    def mark_incomplete(self, batch, progress=None):
456        """
457        Mark the given batch as "incomplete" (aka. pending).  This usually is
458        just a matter of clearing the
459        :attr:`~rattail.db.model.batch.BatchMixin.complete` flag for the batch,
460        with the idea that this should "thaw" the batch so that it may be
461        further updated, i.e. it's not yet ready to execute.
462
463        Each handler is of course free to expound on this idea, or to add extra
464        logic to this "event" of marking a batch incomplete.
465        """
466        batch.complete = False
467
468    def why_not_execute(self, batch):
469        """
470        This method should return a string indicating the reason why the given
471        batch should not be considered executable.  By default it returns
472        ``None`` which means the batch *is* to be considered executable.
473
474        Note that it is assumed the batch has not already been executed, since
475        execution is globally prevented for such batches.
476        """
477
478    def executable(self, batch):
479        """
480        This method should return a boolean indicating whether or not execution
481        should be allowed for the batch, given its current condition.  The
482        default simply returns ``True`` but you may override as needed.
483
484        Note that this (currently) only affects the enabled/disabled state of
485        the Execute button within the Tailbone batch view.
486        """
487        if batch is None:
488            return True
489        if batch.executed:
490            return False
491        if self.why_not_execute(batch):
492            return False
493        return True
494
495    def auto_executable(self, batch):
496        """
497        Must return a boolean indicating whether the given bath is eligible for
498        "automatic" execution, i.e. immediately after batch is created.
499        """
500        return False
501
502    def do_execute(self, batch, user, progress=None, **kwargs):
503        """
504        Perform final execution for the batch.  What that means for any given
505        batch, will vary greatly.
506
507        Note that callers *should* use this method, but custom batch handlers
508        should *not* override this method.  Conversely, custom handlers
509        *should* override the :meth:`~execute()` method, but callers should
510        *not* use that one directly.
511        """
512        result = self.execute(batch, user=user, progress=progress, **kwargs)
513        if not result:
514            return False
515        batch.executed = make_utc()
516        batch.executed_by = user
517        return result
518
519    def execute(self, batch, progress=None, **kwargs):
520        """
521        Execute the given batch, with given progress and kwargs.  That is an
522        intentionally generic statement, the meaning of which must be further
523        defined by the handler subclass since default is ``NotImplementedError``.
524
525        Note that callers should *not* use this method, but custom batch
526        handlers *should* override this method.  Conversely, custom handlers
527        should *not* override the :meth:`~do_execute()` method, but callers
528        *should* use that one directly.
529        """
530        raise NotImplementedError
531
532    def execute_many(self, batches, progress=None, **kwargs):
533        """
534        Execute a set of batches, with given progress and kwargs.  Default
535        behavior is to simply execute each batch in succession.  Any batches
536        which are already executed are skipped.
537        """
538        now = make_utc()
539        for batch in batches:
540            if not batch.executed:
541                self.execute(batch, progress=progress, **kwargs)
542                batch.executed = now
543                batch.executed_by = kwargs['user']
544        return True
545
546    def delete(self, batch, delete_all_data=True, progress=None, **kwargs):
547        """
548        Delete all data for the batch, including any related (e.g. row)
549        records, as well as files on disk etc.  This method should *not* delete
550        the batch itself however.
551
552        :param delete_all_data: Flag indicating whether *all* data should be
553           deleted.  You should probably set this to ``False`` if in dry-run
554           mode, since deleting *all* data often implies deleting files from
555           disk, which is not transactional and therefore can't be rolled back.
556        """
557        if delete_all_data:
558            if hasattr(batch, 'delete_data'):
559                batch.delete_data(self.config)
560        if hasattr(batch, 'data_rows'):
561            del batch.data_rows[:]
562
563    def setup_clone(self, oldbatch, progress=None):
564        """
565        Perform any setup (caching etc.) necessary for cloning batch.  Note
566        that the ``oldbatch`` arg is the "old" batch, i.e. the one from which a
567        clone is to be created.
568        """
569
570    def teardown_clone(self, newbatch, progress=None):
571        """
572        Perform any teardown (cleanup etc.) necessary after cloning a batch.
573        Note that the ``newbatch`` arg is the "new" batch, i.e. the one which
574        was just created by cloning the old batch.
575        """
576
577    def clone(self, oldbatch, created_by, progress=None):
578        """
579        Clone the given batch as a new batch, and return the new batch.
580        """
581        self.setup_clone(oldbatch, progress=progress)
582        batch_class = self.batch_model_class
583        batch_mapper = orm.class_mapper(batch_class)
584
585        newbatch = batch_class()
586        newbatch.created_by = created_by
587        newbatch.rowcount = 0
588        for name in batch_mapper.columns.keys():
589            if name not in ('uuid', 'id', 'created', 'created_by_uuid', 'rowcount', 'executed', 'executed_by_uuid'):
590                setattr(newbatch, name, getattr(oldbatch, name))
591
592        session = orm.object_session(oldbatch)
593        session.add(newbatch)
594        session.flush()
595
596        row_class = newbatch.row_class
597        row_mapper = orm.class_mapper(row_class)
598
599        def clone_row(oldrow, i):
600            newrow = self.clone_row(oldrow)
601            self.add_row(newbatch, newrow)
602
603        self.progress_loop(clone_row, oldbatch.data_rows, progress,
604                           message="Cloning data rows for new batch")
605
606        self.refresh_batch_status(newbatch)
607        self.teardown_clone(newbatch, progress=progress)
608        return newbatch
609
610    def clone_row(self, oldrow):
611        row_class = self.batch_model_class.row_class
612        row_mapper = orm.class_mapper(row_class)
613        newrow = row_class()
614        for name in row_mapper.columns.keys():
615            if name not in ('uuid', 'batch_uuid', 'sequence'):
616                setattr(newrow, name, getattr(oldrow, name))
617        return newrow
618
619    def cache_model(self, session, model, **kwargs):
620        return cache_model(session, model, **kwargs)
621
622
623def get_batch_types(config):
624    """
625    Returns the list of available batch type keys.
626    """
627    model = config.get_model()
628
629    keys = []
630    for name in dir(model):
631        if name == 'BatchMixin':
632            continue
633        obj = getattr(model, name)
634        if isinstance(obj, type):
635            if issubclass(obj, model.Base):
636                if issubclass(obj, model.BatchMixin):
637                    keys.append(obj.batch_key)
638
639    keys.sort()
640    return keys
641
642
643def get_batch_handler(config, batch_key, default=None, error=True):
644    """
645    Returns a batch handler object corresponding to the given batch key.
646    """
647    spec = config.get('rattail.batch', '{}.handler'.format(batch_key), default=default)
648    if error and not spec:
649        raise ValueError("handler spec not found for batch type: {}".format(batch_key))
650    handler = load_object(spec)(config)
651    return handler
Note: See TracBrowser for help on using the repository browser.