source: rattail/rattail/batch/handlers.py @ e7c6c31

Last change on this file since e7c6c31 was e7c6c31, checked in by Lance Edgar <lance@…>, 11 months ago

Record execution kwargs as special params, when executing batch

  • Property mode set to 100644
File size: 37.4 KB
Line 
1# -*- coding: utf-8; -*-
2################################################################################
3#
4#  Rattail -- Retail Software Framework
5#  Copyright © 2010-2022 Lance Edgar
6#
7#  This file is part of Rattail.
8#
9#  Rattail is free software: you can redistribute it and/or modify it under the
10#  terms of the GNU General Public License as published by the Free Software
11#  Foundation, either version 3 of the License, or (at your option) any later
12#  version.
13#
14#  Rattail is distributed in the hope that it will be useful, but WITHOUT ANY
15#  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
16#  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
17#  details.
18#
19#  You should have received a copy of the GNU General Public License along with
20#  Rattail.  If not, see <http://www.gnu.org/licenses/>.
21#
22################################################################################
23"""
24Data Batch Handlers
25"""
26
27from __future__ import unicode_literals, absolute_import
28
29import os
30import shutil
31import datetime
32import warnings
33import logging
34
35import json
36import six
37from sqlalchemy import orm
38
39from rattail.db import model, api
40from rattail.core import Object
41from rattail.gpc import GPC
42from rattail.barcodes import upce_to_upca
43from rattail.db.cache import cache_model
44from rattail.time import localtime, make_utc
45from rattail.util import progress_loop, load_object
46
47
48log = logging.getLogger(__name__)
49
50
51class BatchHandler(object):
52    """
53    Base class and partial default implementation for batch handlers.  It is
54    expected that all batch handlers will ultimately inherit from this base
55    class, therefore it defines the implementation "interface" loosely
56    speaking.  Custom batch handlers are welcome to supplement or override this
57    as needed, and in fact must do so for certain aspects.
58    """
59    populate_batches = False
60    populate_with_versioning = True
61
62    refresh_with_versioning = True
63    repopulate_when_refresh = False
64
65    execute_with_versioning = True
66    delete_with_versioning = True
67
68    def __init__(self, config):
69        self.config = config
70        self.app = self.config.get_app()
71        self.enum = config.get_enum()
72        self.model = config.get_model()
73
74    @property
75    def batch_model_class(self):
76        """
77        Reference to the data model class of the batch type for which this
78        handler is responsible,
79        e.g. :class:`~rattail.db.model.batch.labels.LabelBatch`.  Each handler
80        must define this (or inherit from one that does).
81        """
82        raise NotImplementedError("You must set the 'batch_model_class' attribute "
83                                  "for class '{}'".format(self.__class__.__name__))
84
85    @property
86    def batch_key(self):
87        """
88        The "batch type key" for the handler, e.g. ``'labels'``.  This is not
89        meant to uniquely identify the handler itself, but rather the *type* of
90        batch which it handles.  Therefore multiple handlers may be defined
91        which share the same ``batch_key`` - although in practice usually each
92        app will need only one handler per batch type.
93
94        If the handler doesn't define this, it will be obtained from the
95        ``batch_key`` attribute of the :attr:`batch_model_class` attribute.
96        """
97        return self.batch_model_class.batch_key
98
99    def get_model_title(self):
100        return self.batch_model_class.get_model_title()
101
102    def allow_versioning(self, action):
103        if action == 'populate':
104            return self.populate_with_versioning
105        if action == 'refresh':
106            return self.refresh_with_versioning
107        if action == 'execute':
108            return self.execute_with_versioning
109        if action == 'delete':
110            return self.delete_with_versioning
111        raise NotImplementedError("unknown batch action: {}".format(action))
112
113    def is_mutable(self, batch):
114        """
115        Returns a boolean indicating whether or not *any* modifications are to
116        be allowed for the given batch.  By default this returns ``False`` if
117        the batch is already executed, or has been marked complete; otherwise
118        returns ``True``.
119        """
120        if batch.executed:
121            return False
122        if batch.complete:
123            return False
124        return True
125
126    def consume_batch_id(self, session, as_str=False):
127        """
128        Consumes a new batch ID from the generator, and returns it.
129
130        :param session: Current session for Rattail DB.
131
132        :param as_str: Flag indicating whether the return value should be a
133           string, as opposed to the default of integer.
134
135        :returns: Batch ID as integer, or zero-padded string of 8 chars.
136        """
137        batch_id = self.app.next_counter_value(session, 'batch_id')
138        if as_str:
139            return '{:08d}'.format(batch_id)
140        return batch_id
141
142    def make_basic_batch(self, session, user=None, progress=None, **kwargs):
143        """
144        Make a new "basic" batch, with no customization beyond what is provided
145        by ``kwargs``, which are passed directly to the batch class constructor.
146
147        Note that the new batch instance will be added to the provided session,
148        which will also be flushed.
149
150        Callers should use :meth:`make_batch()` instead of this method.
151        """
152        kwargs.setdefault('rowcount', 0)
153        kwargs.setdefault('complete', False)
154
155        # we used just let the postgres sequence auto-generate the id,
156        # but now we are trying to support more than just postgres,
157        # and so we consume a batch id using shared logic which can
158        # accommodate more than just postgres
159        if 'id' not in kwargs:
160            kwargs['id'] = self.consume_batch_id(session)
161
162        # try to provide default creator
163        if user and 'created_by' not in kwargs and 'created_by_uuid' not in kwargs:
164            kwargs['created_by'] = user
165
166        batch = self.batch_model_class(**kwargs)
167        session.add(batch)
168        session.flush()
169        return batch
170
171    def make_batch(self, session, progress=None, **kwargs):
172        """
173        Make and return a new batch instance.
174
175        This is the method which callers should use.  It invokes
176        :meth:`make_basic_batch()` to actually create the new batch instance,
177        and then :meth:`init_batch()` to perform any extra initialization for
178        it.  Note that the batch returned will *not* yet be fully populated.
179        """
180        batch = self.make_basic_batch(session, progress=progress, **kwargs)
181        kwargs['session'] = session
182        self.init_batch(batch, progress=progress, **kwargs)
183        return batch
184
185    def init_batch(self, batch, progress=None, **kwargs):
186        """
187        Perform extra initialization for the given batch, in whatever way might
188        make sense.  Default of course is to do nothing here; handlers are free
189        to override as needed.
190
191        Note that initial population of a batch should *not* happen here; see
192        :meth:`populate()` for a place to define that logic.
193        """
194
195    def make_row(self, **kwargs):
196        """
197        Make a new row for the batch.  Note however that the row will **not**
198        be added to the batch; that should be done with :meth:`add_row()`.
199
200        :returns: A new row object, which does *not* yet belong to any batch.
201        """
202        return self.batch_model_class.row_class(**kwargs)
203
204    def add_row(self, batch, row):
205        """
206        Add the given row to the given batch.  This assumes it is a *new* row
207        which does not yet belong to any batch.  This logic performs the
208        following steps:
209
210        The row is officially added to the batch, and is immediately
211        "refreshed" via :meth:`refresh_row()`.
212
213        The row is then examined to see if it has been marked as "removed" by
214        the refresh.  If it was *not* removed then the batch's cached
215        ``rowcount`` is incremented, and the :meth:`after_add_row()` hook is
216        invoked.
217        """
218        session = orm.object_session(batch)
219        with session.no_autoflush:
220            batch.data_rows.append(row)
221            self.refresh_row(row)
222        if not row.removed:
223            batch.rowcount = (batch.rowcount or 0) + 1
224            self.after_add_row(batch, row)
225
226    def after_add_row(self, batch, row):
227        """
228        Event hook, called immediately after the given row has been "properly"
229        added to the batch.  This is a good place to update totals for the
230        batch, to account for the new row, etc.
231        """
232
233    def purge_batches(self, session, before=None, before_days=90,
234                      dry_run=False, delete_all_data=None,
235                      progress=None, **kwargs):
236        """
237        Purge all batches which were executed prior to a given date.
238
239        :param before: If provided, must be a timezone-aware datetime object.
240           If not provided, it will be calculated from the current date, using
241           ``before_days``.
242
243        :param before_days: Number of days before the current date, to be used
244           as the cutoff date if ``before`` is not specified.
245
246        :param dry_run: Flag indicating that this is a "dry run" and all logic
247           involved should be (made) aware of that fact.
248
249        :param delete_all_data: Flag indicating whether *all* data should be
250           deleted for each batch being purged.  This flag is passed along to
251           :meth:`delete()`; see that for more info.  NOTE: This flag should
252           probably be deprecated, but so far has not been...but ``dry_run``
253           should be preferred for readability etc.
254
255        :returns: Integer indicating the number of batches purged.
256        """
257        if delete_all_data and dry_run:
258            raise ValueError("You can enable (n)either of `dry_run` or "
259                             "`delete_all_data` but both cannot be True")
260        delete_all_data = not dry_run
261
262        if not before:
263            before = localtime(self.config).date() - datetime.timedelta(days=before_days)
264            before = datetime.datetime.combine(before, datetime.time(0))
265            before = localtime(self.config, before)
266
267        log.info("will purge '%s' batches, executed before %s",
268                 self.batch_key, before.date())
269
270        old_batches = session.query(self.batch_model_class)\
271                             .filter(self.batch_model_class.executed < before)\
272                             .options(orm.joinedload(self.batch_model_class.data_rows))\
273                             .all()
274        log.info("found %s batches to purge", len(old_batches))
275        result = Object()
276        result.purged = 0
277
278        def purge(batch, i):
279            self.do_delete(batch, dry_run=dry_run)
280            result.purged += 1
281            if i % 5 == 0:
282                session.flush()
283
284        self.progress_loop(purge, old_batches, progress,
285                           message="Purging old batches")
286
287        session.flush()
288        if old_batches:
289            log.info("%spurged %s '%s' batches",
290                     "(would have) " if dry_run else "",
291                     result.purged, self.batch_key)
292        return result.purged
293
294    @property
295    def root_datadir(self):
296        """
297        The absolute path of the root folder in which data for this particular
298        type of batch is stored.  The structure of this path is as follows:
299
300        .. code-block:: none
301
302           /{root_batch_data_dir}/{batch_type_key}
303
304        * ``{root_batch_data_dir}`` - Value of the 'batch.files' option in the
305          [rattail] section of config file.
306        * ``{batch_type_key}`` - Unique key for the type of batch it is.
307
308        .. note::
309           While it is likely that the data folder returned by this method
310           already exists, this method does not guarantee it.
311        """
312        return self.config.batch_filedir(self.batch_key)
313
314    def datadir(self, batch):
315        """
316        Returns the absolute path of the folder in which the batch's source
317        data file(s) resides.  Note that the batch must already have been
318        persisted to the database.  The structure of the path returned is as
319        follows:
320
321        .. code-block:: none
322
323           /{root_datadir}/{uuid[:2]}/{uuid[2:]}
324
325        * ``{root_datadir}`` - Value returned by :meth:`root_datadir()`.
326        * ``{uuid[:2]}`` - First two characters of batch UUID.
327        * ``{uuid[2:]}`` - All batch UUID characters *after* the first two.
328
329        .. note::
330           While it is likely that the data folder returned by this method
331           already exists, this method does not guarantee any such thing.  It
332           is typically assumed that the path will have been created by a
333           previous call to :meth:`make_batch()` however.
334        """
335        return os.path.join(self.root_datadir, batch.uuid[:2], batch.uuid[2:])
336
337    def make_datadir(self, batch):
338        """
339        Returns the data folder specific to the given batch, creating if necessary.
340        """
341        datadir = self.datadir(batch)
342        if not os.path.exists(datadir):
343            os.makedirs(datadir)
344        return datadir
345
346    # TODO: remove default attr?
347    def set_input_file(self, batch, path, attr='filename'):
348        """
349        Assign the data file found at ``path`` to the batch.  This overwrites
350        the given attribute (``attr``) of the batch and places a copy of the
351        data file in the batch's data folder.
352        """
353        datadir = self.make_datadir(batch)
354        filename = os.path.basename(path)
355        shutil.copyfile(path, os.path.join(datadir, filename))
356        setattr(batch, attr, filename)
357
358    def should_populate(self, batch):
359        """
360        Must return a boolean indicating whether the given batch should be
361        populated from an initial data source, i.e. at time of batch creation.
362        Override this method if you need to inspect the batch in order to
363        determine whether the populate step is needed.  Default behavior is to
364        simply return the value of :attr:`populate_batches`.
365        """
366        return self.populate_batches
367
368    def setup_populate(self, batch, progress=None):
369        """
370        Perform any setup (caching etc.) necessary for populating a batch.
371        """
372
373    def teardown_populate(self, batch, progress=None):
374        """
375        Perform any teardown (cleanup etc.) necessary after populating a batch.
376        """
377
378    def do_populate(self, batch, user, progress=None):
379        """
380        Perform initial population for the batch, i.e. fill it with data rows.
381        Where the handler obtains the data to do this, will vary greatly.
382
383        Note that callers *should* use this method, but custom batch handlers
384        should *not* override this method.  Conversely, custom handlers
385        *should* override the :meth:`~populate()` method, but callers should
386        *not* use that one directly.
387        """
388        self.setup_populate(batch, progress=progress)
389        self.populate(batch, progress=progress)
390        self.teardown_populate(batch, progress=progress)
391        self.refresh_batch_status(batch)
392        return True
393
394    def populate(self, batch, progress=None):
395        """
396        Populate the batch with initial data rows.  It is assumed that the data
397        source to be used will be known by inspecting various properties of the
398        batch itself.
399
400        Note that callers should *not* use this method, but custom batch
401        handlers *should* override this method.  Conversely, custom handlers
402        should *not* override the :meth:`~do_populate()` method, but callers
403        *should* use that one directly.
404        """
405        raise NotImplementedError("Please implement `{}.populate()` method".format(self.__class__.__name__))
406
407    def refreshable(self, batch):
408        """
409        This method should return a boolean indicating whether or not the
410        handler supports a "refresh" operation for the batch, given its current
411        condition.  The default assumes a refresh is allowed unless the batch
412        has already been executed.
413        """
414        if batch.executed:
415            return False
416        return True
417
418    def progress_loop(self, *args, **kwargs):
419        return progress_loop(*args, **kwargs)
420
421    def setup_refresh(self, batch, progress=None):
422        """
423        Perform any setup (caching etc.) necessary for refreshing a batch.
424        """
425
426    def teardown_refresh(self, batch, progress=None):
427        """
428        Perform any teardown (cleanup etc.) necessary after refreshing a batch.
429        """
430
431    def do_refresh(self, batch, user, progress=None):
432        """
433        Perform a full data refresh for the batch, i.e. update any data which
434        may have become stale, etc.
435
436        Note that callers *should* use this method, but custom batch handlers
437        should *not* override this method.  Conversely, custom handlers
438        *should* override the :meth:`~refresh()` method, but callers should
439        *not* use that one directly.
440        """
441        self.refresh(batch, progress=progress)
442        return True
443
444    def refresh(self, batch, progress=None):
445        """
446        Perform a full data refresh for the batch.  What exactly this means will
447        depend on the type of batch, and specific handler logic.
448
449        Generally speaking this refresh is meant to use queries etc. to obtain
450        "fresh" data for the batch (header) and all its rows.  In most cases
451        certain data is expected to be "core" to the batch and/or rows, and
452        such data will be left intact, with all *other* data values being
453        re-calculated and/or reset etc.
454
455        Note that callers should *not* use this method, but custom batch
456        handlers *should* override this method.  Conversely, custom handlers
457        should *not* override the :meth:`~do_refresh()` method, but callers
458        *should* use that one directly.
459        """
460        session = orm.object_session(batch)
461        self.setup_refresh(batch, progress=progress)
462        if self.repopulate_when_refresh:
463            del batch.data_rows[:]
464            batch.rowcount = 0
465            session.flush()
466            self.populate(batch, progress=progress)
467        else:
468            batch.rowcount = 0
469
470            def refresh(row, i):
471                with session.no_autoflush:
472                    self.refresh_row(row)
473                if not row.removed:
474                    batch.rowcount += 1
475
476            self.progress_loop(refresh, batch.active_rows(), progress,
477                               message="Refreshing batch data rows")
478        self.refresh_batch_status(batch)
479        self.teardown_refresh(batch, progress=progress)
480        return True
481
482    def refresh_many(self, batches, user=None, progress=None):
483        """
484        Refresh a set of batches, with given progress.  Default behavior is to
485        simply refresh each batch in succession.  Any batches which are already
486        executed are skipped.
487
488        Handlers may have to override this method if "grouping" or other
489        special behavior is needed.
490        """
491        needs_refresh = [batch for batch in batches
492                         if not batch.executed]
493        if not needs_refresh:
494            return
495
496        # TODO: should perhaps try to make the progress indicator reflect the
497        # "total" number of rows across all batches being refreshed?  seems
498        # like that might be more accurate, for the user.  but also harder.
499
500        for batch in needs_refresh:
501            self.do_refresh(batch, user, progress=progress)
502
503    def refresh_row(self, row):
504        """
505        This method will be passed a row object which has already been properly
506        added to a batch, and which has basic required fields already
507        populated.  This method is then responsible for further populating all
508        applicable fields for the row, based on current data within the
509        appropriate system(s).
510
511        Note that in some cases this method may be called multiple times for
512        the same row, e.g. once when first creating the batch and then later
513        when a user explicitly refreshes the batch.  The method logic must
514        account for this possibility.
515        """
516
517    def refresh_product_basics(self, row):
518        """
519        This method will refresh the "basic" product info for a row.  It
520        assumes that the row is derived from
521        :class:`~rattail.db.model.batch.core.ProductBatchRowMixin` and that
522        ``row.product`` is already set to a valid product.
523        """
524        product = getattr(row, 'product', None)
525        if not product:
526            return
527
528        row.item_id = product.item_id
529        row.upc = product.upc
530        row.brand_name = six.text_type(product.brand or "")
531        row.description = product.description
532        row.size = product.size
533
534        department = product.department
535        row.department_number = department.number if department else None
536        row.department_name = department.name if department else None
537
538        subdepartment = product.subdepartment
539        row.subdepartment_number = subdepartment.number if subdepartment else None
540        row.subdepartment_name = subdepartment.name if subdepartment else None
541
542    def quick_entry(self, session, batch, entry):
543        """
544        Handle a "quick entry" value, e.g. from user input.  Most frequently this
545        value would represent a UPC or similar "ID" value for e.g. a product record,
546        and the handler's duty would be to either locate a corresponding row within
547        the batch (if one exists), or else add a new row to the batch.
548
549        In any event this method can be customized and in fact has no default
550        behavior, so must be defined by a handler.
551
552        :param session: Database sesssion.
553
554        :param batch: Batch for which the quick entry is to be handled.  Note that
555           this batch is assumed to belong to the given ``session``.
556
557        :param entry: String value to be handled.  This is generally assumed to
558           be from user input (e.g. UPC scan field) but may not always be.
559
560        :returns: New or existing "row" object, for the batch.
561        """
562        raise NotImplementedError
563
564    def locate_product_for_entry(self, session, entry, **kwargs):
565        """
566        TODO: Deprecated; use ProductsHandler method instead.
567        """
568        # don't bother if we're given empty "entry" value
569        if not entry:
570            return
571
572        products_handler = self.app.get_products_handler()
573        return products_handler.locate_product_for_entry(session, entry,
574                                                         **kwargs)
575
576    def remove_row(self, row):
577        """
578        Remove the given row from its batch, and update the batch accordingly.
579        How exactly the row is "removed" is up to this method.  Default is to
580        set the row's ``removed`` flag, then invoke the
581        :meth:`refresh_batch_status()` method.
582
583        Note that callers should *not* use this method, but custom batch
584        handlers *should* override this method.  Conversely, custom handlers
585        should *not* override the :meth:`do_remove_row()` method, but callers
586        *should* use that one directly.
587        """
588        batch = row.batch
589        row.removed = True
590        self.refresh_batch_status(batch)
591
592    def do_remove_row(self, row):
593        """
594        Remove the given row from its batch, and update the batch accordingly.
595        Uses the following logic:
596
597        If the row's ``removed`` flag is already set, does nothing and returns
598        immediately.
599
600        Otherwise, it invokes :meth:`remove_row()` and then decrements the
601        batch ``rowcount`` attribute.
602
603        Note that callers *should* use this method, but custom batch handlers
604        should *not* override this method.  Conversely, custom handlers
605        *should* override the :meth:`remove_row()` method, but callers should
606        *not* use that one directly.
607        """
608        if row.removed:
609            return
610        self.remove_row(row)
611        batch = row.batch
612        if batch.rowcount is not None:
613            batch.rowcount -= 1
614
615    def refresh_batch_status(self, batch):
616        """
617        Update the batch status, as needed.  This method does nothing by
618        default, but may be overridden if the overall batch status needs to be
619        updated according to the status of its rows.  This method may be
620        invoked whenever rows are added, removed, updated etc.
621        """
622
623    def write_worksheet(self, batch, progress=None):
624        """
625        Write a worksheet file, to be downloaded by the user.  Must return the
626        file path.
627        """
628        raise NotImplementedError("Please define logic for `{}.write_worksheet()`".format(
629            self.__class__.__name__))
630
631    def update_from_worksheet(self, batch, path, progress=None):
632        """
633        Save the given file to a batch-specific location, then update the
634        batch data from the file contents.
635        """
636        raise NotImplementedError("Please define logic for `{}.update_from_worksheet()`".format(
637            self.__class__.__name__))
638
639    def mark_complete(self, batch, progress=None):
640        """
641        Mark the given batch as "complete".  This usually is just a matter of
642        setting the :attr:`~rattail.db.model.batch.BatchMixin.complete` flag
643        for the batch, with the idea that this should "freeze" the batch so
644        that another user can verify its state before finally executing it.
645
646        Each handler is of course free to expound on this idea, or to add extra
647        logic to this "event" of marking a batch complete.
648        """
649        batch.complete = True
650
651    def mark_incomplete(self, batch, progress=None):
652        """
653        Mark the given batch as "incomplete" (aka. pending).  This usually is
654        just a matter of clearing the
655        :attr:`~rattail.db.model.batch.BatchMixin.complete` flag for the batch,
656        with the idea that this should "thaw" the batch so that it may be
657        further updated, i.e. it's not yet ready to execute.
658
659        Each handler is of course free to expound on this idea, or to add extra
660        logic to this "event" of marking a batch incomplete.
661        """
662        batch.complete = False
663
664    def why_not_execute(self, batch):
665        """
666        This method should inspect the given batch and, if there is a reason
667        that execution should *not* be allowed for it, the method should return
668        a text string indicating that reason.  It should return ``None`` if no
669        such reason could be identified, and execution should be allowed.
670
671        Note that it is assumed the batch has not already been executed, since
672        execution is globally prevented for such batches.  In other words you
673        needn't check for that as a possible reason not to execute.
674        """
675
676    def executable(self, batch):
677        """
678        This method should return a boolean indicating whether or not execution
679        should be allowed for the batch, given its current condition.
680
681        While you may override this method, you are encouraged to override
682        :meth:`why_not_execute()` instead.  Default logic for this method is as
683        follows:
684
685        If the batch is ``None`` then the caller simply wants to know if "any"
686        batch may be executed, so we return ``True``.
687
688        If the batch has already been executed then we return ``False``.
689
690        If the :meth:`why_not_execute()` method returns a value, then we assume
691        execution is not allowed and return ``False``.
692
693        Finally we will return ``True`` if none of the above rules matched.
694        """
695        if batch is None:
696            return True
697        if batch.executed:
698            return False
699        if self.why_not_execute(batch):
700            return False
701        return True
702
703    def auto_executable(self, batch):
704        """
705        Must return a boolean indicating whether the given bath is eligible for
706        "automatic" execution, i.e. immediately after batch is created.
707        """
708        return False
709
710    def describe_execution(self, batch, **kwargs):
711        """
712        This method should essentially return some text describing briefly what
713        will happen when the given batch is executed.
714
715        :param batch: The batch in question, which is a candidate for
716           execution.
717
718        :returns: String value describing the batch execution.
719        """
720
721    def do_execute(self, batch, user, progress=None, **kwargs):
722        """
723        Perform final execution for the batch.  What that means for any given
724        batch, will vary greatly.
725
726        Note that callers *should* use this method, but custom batch handlers
727        should *not* override this method.  Conversely, custom handlers
728        *should* override the :meth:`~execute()` method, but callers should
729        *not* use that one directly.
730        """
731        # make sure we declare who's responsible, if we can
732        # TODO: seems like if caller already knows user, they should
733        # have already done this.  and probably bad form to do it here
734        session = self.app.get_session(batch)
735        session.set_continuum_user(user)
736
737        result = self.execute(batch, user=user, progress=progress, **kwargs)
738        if not result:
739            return False
740
741        batch.executed = make_utc()
742        batch.executed_by = user
743
744        # record the execution kwargs within batch params, if there
745        # were any.  this is mostly for troubleshooting after the fact
746        if kwargs:
747            try:
748                # first make sure kwargs are JSON-safe
749                json.dumps(kwargs)
750            except:
751                # TODO: may need to lower log level if this is common,
752                # although underlying causes are hopefully easy to fix
753                log.exception("kwargs are not JSON-safe: %s", kwargs)
754            else:
755                batch.set_param('_executed_with_kwargs_', kwargs)
756
757        return result
758
759    def get_effective_rows(self, batch):
760        """
761        Should return the set of rows from the given batch which are
762        considered "effective" - i.e. when the batch is executed,
763        these rows should be processed whereas the remainder should
764        not.
765
766        :param batch: A
767           :class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatch`
768           instance.
769
770        :returns: List of
771           :class:`~rattail.db.model.batch.vendorcatalog.VendorCatalogBatchRow`
772           instances.
773        """
774        return batch.active_rows()
775
776    def execute(self, batch, progress=None, **kwargs):
777        """
778        Execute the given batch, according to the given kwargs.  This is really
779        where the magic happens, although each handler must define that magic,
780        since the default logic does nothing at all.
781
782        Note that callers should *not* use this method, but custom batch
783        handlers *should* override this method.  Conversely, custom handlers
784        should *not* override the :meth:`~do_execute()` method, but callers
785        *should* use that one directly.
786        """
787
788    def execute_many(self, batches, progress=None, **kwargs):
789        """
790        Execute a set of batches, with given progress and kwargs.  Default
791        behavior is to simply execute each batch in succession.  Any batches
792        which are already executed are skipped.
793
794        Handlers may have to override this method if "grouping" or other
795        special behavior is needed.
796        """
797        now = make_utc()
798        for batch in batches:
799            if not batch.executed:
800                self.execute(batch, progress=progress, **kwargs)
801                batch.executed = now
802                batch.executed_by = kwargs['user']
803        return True
804
805    def do_delete(self, batch, dry_run=False, progress=None, **kwargs):
806        """
807        Totally delete the given batch.  This includes deleting the batch
808        itself, any rows and "extra" data such as files.
809
810        Note that callers *should* use this method, but custom batch handlers
811        should *not* override this method.  Conversely, custom handlers
812        *should* override the :meth:`~delete()` method, but callers should
813        *not* use that one directly.
814        """
815        session = orm.object_session(batch)
816
817        if 'delete_all_data' in kwargs:
818            warnings.warn("The 'delete_all_data' kwarg is not supported for "
819                          "this method; please use 'dry_run' instead",
820                          DeprecationWarning)
821        kwargs['delete_all_data'] = not dry_run
822
823        self.delete(batch, progress=progress, **kwargs)
824        session.delete(batch)
825
826    def delete(self, batch, delete_all_data=True, progress=None, **kwargs):
827        """
828        Delete all data for the batch, including any related (e.g. row)
829        records, as well as files on disk etc.  This method should *not* delete
830        the batch itself however.
831
832        Note that callers should *not* use this method, but custom batch
833        handlers *should* override this method.  Conversely, custom handlers
834        should *not* override the :meth:`~do_delete()` method, but callers
835        *should* use that one directly.
836
837        :param delete_all_data: Flag indicating whether *all* data should be
838           deleted.  You should probably set this to ``False`` if in dry-run
839           mode, since deleting *all* data often implies deleting files from
840           disk, which is not transactional and therefore can't be rolled back.
841        """
842        if delete_all_data:
843            self.delete_extra_data(batch, progress=progress)
844
845        # delete all rows from batch, one by one.  maybe would be nicer if we
846        # could delete all in one fell swoop, but sometimes "extension" row
847        # records might exist, and can get FK constraint errors
848        # TODO: in other words i don't even know why this is necessary.  seems
849        # to me that one fell swoop should not incur FK errors
850        if hasattr(batch, 'data_rows'):
851            session = orm.object_session(batch)
852
853            def delete(row, i):
854                session.delete(row)
855                if i % 200 == 0:
856                    session.flush()
857
858            self.progress_loop(delete, batch.data_rows, progress,
859                               message="Deleting rows from batch")
860            session.flush()
861
862            # even though we just deleted all rows, we must also "remove" all
863            # rows explicitly from the batch; otherwise when the batch itself
864            # is deleted, SQLAlchemy may complain about an unexpected number of
865            # rows being deleted
866            del batch.data_rows[:]
867
868    def delete_extra_data(self, batch, progress=None, **kwargs):
869        """
870        Delete all "extra" data for the batch.  This method should *not* bother
871        trying to delete the batch itself, or rows thereof.  It typically is
872        only concerned with deleting extra files on disk, related to the batch.
873        """
874        path = self.config.batch_filepath(self.batch_key, batch.uuid)
875        if os.path.exists(path):
876            shutil.rmtree(path)
877
878    def setup_clone(self, oldbatch, progress=None):
879        """
880        Perform any setup (caching etc.) necessary for cloning batch.  Note
881        that the ``oldbatch`` arg is the "old" batch, i.e. the one from which a
882        clone is to be created.
883        """
884
885    def teardown_clone(self, newbatch, progress=None):
886        """
887        Perform any teardown (cleanup etc.) necessary after cloning a batch.
888        Note that the ``newbatch`` arg is the "new" batch, i.e. the one which
889        was just created by cloning the old batch.
890        """
891
892    def clone(self, oldbatch, created_by, progress=None):
893        """
894        Clone the given batch as a new batch, and return the new batch.
895        """
896        self.setup_clone(oldbatch, progress=progress)
897        batch_class = self.batch_model_class
898        batch_mapper = orm.class_mapper(batch_class)
899
900        newbatch = batch_class()
901        newbatch.created_by = created_by
902        newbatch.rowcount = 0
903        for name in batch_mapper.columns.keys():
904            if name not in ('uuid', 'id', 'created', 'created_by_uuid', 'rowcount', 'executed', 'executed_by_uuid'):
905                setattr(newbatch, name, getattr(oldbatch, name))
906
907        session = orm.object_session(oldbatch)
908        session.add(newbatch)
909        session.flush()
910
911        row_class = newbatch.row_class
912        row_mapper = orm.class_mapper(row_class)
913
914        def clone_row(oldrow, i):
915            newrow = self.clone_row(oldrow)
916            self.add_row(newbatch, newrow)
917
918        self.progress_loop(clone_row, oldbatch.data_rows, progress,
919                           message="Cloning data rows for new batch")
920
921        self.refresh_batch_status(newbatch)
922        self.teardown_clone(newbatch, progress=progress)
923        return newbatch
924
925    def clone_row(self, oldrow):
926        row_class = self.batch_model_class.row_class
927        row_mapper = orm.class_mapper(row_class)
928        newrow = row_class()
929        for name in row_mapper.columns.keys():
930            if name not in ('uuid', 'batch_uuid', 'sequence'):
931                setattr(newrow, name, getattr(oldrow, name))
932        return newrow
933
934    def cache_model(self, session, model, **kwargs):
935        return cache_model(session, model, **kwargs)
936
937
938def get_batch_types(config):
939    """
940    Returns the list of available batch type keys.
941    """
942    model = config.get_model()
943
944    keys = []
945    for name in dir(model):
946        if name == 'BatchMixin':
947            continue
948        obj = getattr(model, name)
949        if isinstance(obj, type):
950            if issubclass(obj, model.Base):
951                if issubclass(obj, model.BatchMixin):
952                    keys.append(obj.batch_key)
953
954    keys.sort()
955    return keys
956
957
958def get_batch_handler(config, batch_key, default=None, error=True): # pragma: no cover
959    warnings.warn("function is deprecated; please use "
960                  "`app.get_batch_handler() instead",
961                  DeprecationWarning)
962    app = config.get_app()
963    return app.get_batch_handler(batch_key, default=default, error=error)
Note: See TracBrowser for help on using the repository browser.