Changeset 5c879a8 in rattail

Timestamp:

11/20/2022 07:35:41 PM (2 months ago)

Author:

Lance Edgar <lance@…>

Branches:

master

Children:

23318f2

Parents:

8cda194

Message:

Add luigi module/class awareness to overnight task launcher

so we can invoke luigi directly instead of relying on script

this also improves logic for how run-n-mail and/or at are used

Location:

rattail

Files:

• commands/luigi.py (modified) (2 diffs)
• luigi/handler.py (modified) (6 diffs)

rattail/commands/luigi.py

@@ -57 +57 @@
                             "determining recipients etc.")
 
+        parser.add_argument('--wait', action='store_true', default=True,
+                            help="Run the task in-process, and wait for it to "
+                            "finish.  This is the default behavior.")
+        parser.add_argument('--no-wait', action='store_false', dest='wait',
+                            help="Do not wait for the task to finish.  This will "
+                            "cause the command to run via the `at` utility, i.e. "
+                            "scheduled to start within the next minute.")
+
         parser.add_argument('--dry-run', action='store_true',
                             help="Log the final command for the task, but do not "
@@ -70 +78 @@
 
         date = args.date or self.app.yesterday()
-        luigi_handler.launch_overnight_task(task, date, with_at=False,
+        luigi_handler.launch_overnight_task(task, date,
                                             email_if_empty=args.email_if_empty,
                                             email_key=args.email_key,
+                                            wait=args.wait,
                                             dry_run=args.dry_run)

rattail/luigi/handler.py

@@ -114 +114 @@
                 'key': key,
                 'description': self.get_overnight_task_setting(key, 'description'),
+                'module': self.get_overnight_task_setting(key, 'module'),
+                'class_name': self.get_overnight_task_setting(key, 'class_name'),
                 'script': self.get_overnight_task_setting(key, 'script'),
                 'notes': self.get_overnight_task_setting(key, 'notes'),
@@ -151 +153 @@
                               email_if_empty=True,
                               email_key=None,
-                              with_at=True,
+                              wait=True,
                               dry_run=False,
                               **kwargs):
@@ -170 +172 @@
            used in determining recipients etc.
 
-        :param with_at: If true (currently the default), the task
-           should be scheduled via the ``at`` command, to begin within
-           the next minute.  (This lets process control return
-           immediately to the caller.)  If false, the task will run
+        :param wait: If true (the default), the task will run
            in-process, and so will begin immediately, but caller must
-           wait for it to complete.  You are encouraged to specify the
-           value you want here, as the default may change in the
-           future.
+           wait for it to complete.  If false, the task will be
+           scheduled via the ``at`` command, to begin within the next
+           minute.  (This lets process control return immediately to
+           the caller.)
 
         :param dry_run: If true, log the final command for the task
@@ -183 +183 @@
         """
         appdir = self.config.appdir()
-
         env = {
+            'PYTHONPATH': appdir,
             'RATTAIL_CONFIG_FILES': os.path.join(appdir, 'silent.conf'),
         }
 
-        cmd = '{} {}'.format(task['script'], date)
-
-        cmd = [os.path.join(sys.prefix, 'bin', 'rattail'),
-               '--config', os.path.join(appdir, 'silent.conf'),
-               '--no-versioning',
-               'run-n-mail',
-               '-S', "Overnight for {}: {}".format(date, task['key']),
-               cmd]
-        if email_key:
-            cmd.extend(['--key', email_key])
-        if not email_if_empty:
-            cmd.append('--skip-if-empty')
-
-        if with_at:
+        # build our command, which will vary per caller request.  by
+        # default we do not use the shell to run command, but in some
+        # cases must (e.g. when invoking `at`)
+        shell = False
+
+        # not waiting means schedule via `at`
+        if not wait:
+
+            # rattail overnight <task>
+            cmd = [
+                '{}/bin/rattail'.format(sys.prefix),
+                '--config', '{}/silent.conf'.format(appdir),
+                '--no-versioning',
+                'overnight', task['key'],
+                '--date={}'.format(date),
+                '--wait',
+            ]
+            if email_key:
+                cmd.extend(['--email-key', email_key])
+            if not email_if_empty:
+                cmd.append('--no-email-if-empty')
+
+            # echo 'rattail overnight <task>' | at now
             cmd = ['echo', shlex_join(cmd)]
             cmd = shlex_join(cmd)
             cmd = "{} | at 'now + 1 minute'".format(cmd)
+            shell = True        # must run command via shell
+
+        elif task['script']:
+            # use whatever task config dictates
+            cmd = self.config.parse_list(task['script'])
+
+        else:
+            # invoke luigi directly, via run-n-mail
+
+            # luigi -m MOD TASK
+            cmd = [
+                '{}/bin/luigi'.format(sys.prefix),
+                '--logging-conf-file', 'logging.conf',
+                '--module', task['module'],
+                task['class_name'],
+                '--date={}'.format(date),
+            ]
+
+            # rattail run-n-mail 'luigi -m MOD TASK'
+            cmd = [
+                os.path.join(sys.prefix, 'bin', 'rattail'),
+                '--config', os.path.join(appdir, 'silent.conf'),
+                '--no-versioning',
+                'run-n-mail',
+                '--subject', "Overnight for {}: {}".format(date, task['key']),
+                shlex_join(cmd),
+            ]
+            if email_key:
+                cmd.extend(['--key', email_key])
+            if not email_if_empty:
+                cmd.append('--skip-if-empty')
 
         # log final command
@@ -213 +253 @@
 
         # run command in subprocess
+        curdir = os.getcwd()
         try:
-            subprocess.check_output(cmd, shell=with_at, env=env,
+            # nb. always chdir to luigi folder, even if not needed
+            os.chdir(os.path.join(appdir, 'luigi'))
+            subprocess.check_output(cmd, shell=shell, env=env,
                                     stderr=subprocess.PIPE)
         except subprocess.CalledProcessError as error:
@@ -221 +264 @@
             log.warning(error.stderr.decode('utf_8'))
             raise
+        finally:
+            # nb. always change back to first dir, in case we're being
+            # called by some long-running process, e.g. web app
+            os.chdir(curdir)
 
     def record_overnight_last_date(self, task, date, session=None, **kwargs):