Enable JupyterLab syntax highlighting, fix tab completion and continuation-prompt hang

pyproject.toml

@@ -38,6 +38,7 @@ content-type = "text/x-rst"
 
 [project.optional-dependencies]
 test = ["pytest", "nbconvert", "jupyter_kernel_test", "nbconvert"]
+jupyterlab = ["jupyterlab-scilab"]
 
 [tool.hatch.build.targets.wheel.shared-data]
 "jupyter-data/share" = "share"

scilab_kernel/check.py

@@ -1,18 +1,19 @@
 import sys
+
 from metakernel import __version__ as mversion
+
 from . import __version__
 from .kernel import ScilabKernel
 
-
 if __name__ == "__main__":
-    print('Scilab kernel v%s' % __version__)
-    print('Metakernel v%s' % mversion)
-    print('Python v%s' % sys.version)
-    print('Python path: %s' % sys.executable)
+    print(f'Scilab kernel v{__version__}')
+    print(f'Metakernel v{mversion}')
+    print(f'Python v{sys.version}')
+    print(f'Python path: {sys.executable}')
     print('\nConnecting to Scilab...')
     try:
         s = ScilabKernel()
         print('Scilab connection established')
         print(s.banner)
-    except Exception as e:
+    except Exception as e:  # noqa: BLE001 -- diagnostic script: report any failure, don't crash with a raw traceback
         print(e)

scilab_kernel/kernel.py

@@ -1,25 +1,75 @@
-from __future__ import print_function, absolute_import
 
 import codecs
+import importlib
 import json
 import os
+import platform
 import re
 import shutil
+import subprocess
 import sys
-import platform
 import tempfile
-import importlib
-import subprocess
+from typing import ClassVar
+
 if importlib.util.find_spec('winreg'):
     import winreg
 from xml.dom import minidom
+from xml.parsers.expat import ExpatError
 
+from IPython.display import SVG, Image
 from metakernel import MetaKernel, ProcessMetaKernel, REPLWrapper, pexpect
 from metakernel.pexpect import which
-from IPython.display import Image, SVG
 
 from . import __version__
 
+# Scilab's terminal control codes (mode/cursor resets, backspace) end up
+# glued to the front of the *first* line of any console output -- used to
+# strip those out of completion results, see get_completions() below.
+_ANSI_ESCAPE_RE = re.compile(r'\x1b\[[0-9;]*[A-Za-z]|\x08')
+
+# Used to detect, after a cell has run, whether Scilab itself flagged an
+# error (undefined variable, wrong dimensions, syntax error, a user's own
+# error(...) call, ...) so its output can be shown in red like any other
+# error, see Write()/_had_scilab_error() below. lasterror() returns [] (an
+# empty matrix, typeof "constant") when nothing went wrong, or the error
+# message (typeof "string", possibly a multi-line string array for a
+# syntax error) otherwise -- and, calling it clears it, so this is safe to
+# run after every non-silent execution without leaking stale state into
+# the next cell.
+_ERROR_MARKER = '<<<SCIKERNELERR>>>'
+_ERROR_FLAG_RE = re.compile(re.escape(_ERROR_MARKER) + r'(\w+)' + re.escape(_ERROR_MARKER))
+_CHECK_ERROR_CMD = f'printf("{_ERROR_MARKER}%s{_ERROR_MARKER}", typeof(lasterror()))'
+
+
+class _ScilabREPLWrapper(REPLWrapper):
+    """A :class:`REPLWrapper` that knows how to escape Scilab's continuation
+    prompt.
+
+    When a cell leaves a block unclosed (``function x=f(y)`` with no
+    ``endfunction``, an ``if``/``for``/``while``/... with no matching
+    ``end``, ...), Scilab drops into a continuation prompt waiting for the
+    rest of the block. REPLWrapper's own recovery for this sends Ctrl-C and
+    waits (up to 30s) for a normal prompt to come back -- but
+    ``scilab-adv-cli`` does not respond to SIGINT while waiting for more
+    input, so that 30s is always spent in full, and the user just sees a
+    generic "Timed out" error after a long pause.
+
+    A bare "end" closes any Scilab block type and returns to the top-level
+    prompt in well under a second; nested unclosed blocks need one "end"
+    per level, so it is sent repeatedly until a normal prompt reappears.
+    """
+
+    _MAX_END_ATTEMPTS = 50
+
+    def interrupt(self, continuation=False):
+        if not continuation:
+            return super().interrupt(continuation=continuation)
+        for _ in range(self._MAX_END_ATTEMPTS):
+            self.sendline("end")
+            if self._expect_prompt(timeout=-1) == 0:
+                break
+        return self.child.before
+
 
 def get_kernel_json():
     """Get the kernel json for the kernel.
@@ -37,10 +87,28 @@ class ScilabKernel(ProcessMetaKernel):
     implementation_version = __version__,
     language = 'scilab'
     language_version = __version__,
-    language_info = {
+    language_info: ClassVar[dict] = {
         'name': 'scilab',
         'file_extension': '.sci',
+        # JupyterLab's CodeMirror editor has no built-in "scilab" language
+        # mode, so real syntax highlighting (correct "//" comments,
+        # keywords, etc.) needs the companion "jupyterlab-scilab" extension
+        # (https://pypi.org/project/jupyterlab-scilab/, `pip install
+        # scilab-kernel[jupyterlab]`), which registers one. Without it,
+        # cells show as plain text in the live editor -- nbconvert/Pygments
+        # export is unaffected either way, since that path uses the 'name'
+        # above, which is a real Pygments lexer ("scilab").
+        #
+        # codemirror_mode is set explicitly (rather than relying on
+        # JupyterLab falling back to matching language_info.name against
+        # its registry) because that fallback goes through a codepath with
+        # a key-name mismatch bug in JupyterLab 4.6's
+        # getMimeTypeByLanguage() for the *mimetype* half of the fallback
+        # (mimetype vs mime, ext vs extensions) -- harmless here only
+        # because the *name* half happens to match first, but not
+        # something to depend on implicitly.
         "mimetype": "text/x-scilab",
+        "codemirror_mode": "scilab",
         "version": __version__,
         'help_links': MetaKernel.help_links,
     }
@@ -106,7 +174,7 @@ def _detect_executable(self):
         # read the windows registry
         if os.name == 'nt':
             try:
-                with winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, "Scilab5.sce\shell\open\command") as key:
+                with winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, r"Scilab5.sce\shell\open\command") as key:
                     cmd : str = winreg.EnumValue(key, 0)[1]
                     executable = cmd.split(r'"')[1].replace("wscilex.exe", "wscilex-cli.exe")
                     self.log.warning('Windows registry binary: ' + executable)
@@ -116,9 +184,9 @@ def _detect_executable(self):
 
         # detect macOS bundle
         if platform.system() == 'Darwin':
-            process = subprocess.run(['mdfind', '-onlyin', '/Applications', 'kMDItemCFBundleIdentifier=org.scilab.modules.jvm.Scilab'], 
-                                 stdout=subprocess.PIPE, 
-                                 universal_newlines=True)
+            process = subprocess.run(['mdfind', '-onlyin', '/Applications', 'kMDItemCFBundleIdentifier=org.scilab.modules.jvm.Scilab'],
+                                 stdout=subprocess.PIPE,
+                                 text=True, check=False)
             bundles = process.stdout
             if len(bundles) > 0:
                 executable = bundles.split('\n', 1)[0] + "/Contents/bin/scilab-adv-cli"
@@ -145,8 +213,9 @@ def makeWrapper(self):
         orig_prompt = r'-[0-9]*->'
         prompt_cmd = None
         change_prompt = None
-        continuation_prompt = '  \>'
+        continuation_prompt = r'  \>'
         self._first = True
+        self._pending_chunk = None
         if os.name == 'nt':
             prompt_cmd = 'printf("-->")'
             echo = False
@@ -157,32 +226,76 @@ def makeWrapper(self):
             echo=echo,
             codec_errors="ignore",
             encoding="utf-8")
-        wrapper = REPLWrapper(child, orig_prompt, change_prompt,
+        wrapper = _ScilabREPLWrapper(child, orig_prompt, change_prompt,
             prompt_emit_cmd=prompt_cmd, echo=echo,
             continuation_prompt_regex=continuation_prompt)
 
         wrapper.child.linesep = '\r\n' if os.name == 'nt' else '\n'
         return wrapper
 
     def Write(self, message):
+        # A Scilab error is always the last thing printed for a command --
+        # execution stops as soon as one occurs -- so whether the *final*
+        # chunk of a command's output should be shown as an error is only
+        # known once the command has finished (see _had_scilab_error()).
+        # Every chunk is therefore held back by one: each new chunk flushes
+        # the previous one (now known not to be the last) as plain output,
+        # and the true last chunk sits in self._pending_chunk until
+        # do_execute_direct() flushes it below, through Write or Error.
         clean_msg = message.strip("\n\r\t")
-        super(ScilabKernel, self).Write(clean_msg)
+        pending, self._pending_chunk = self._pending_chunk, clean_msg
+        if pending is not None:
+            super().Write(pending)
 
     def Print(self, text):
         text = str(text).strip('\x1b[0m').replace('\u0008', '').strip()
         text = [line.strip() for line in text.splitlines()
                 if (not line.startswith(chr(27)))]
         text = '\n'.join(text)
         if text:
-            super(ScilabKernel, self).Print(text)
+            super().Print(text)
+
+    def _had_scilab_error(self):
+        """Whether the command that just ran left an error in Scilab's own
+        error register (undefined variable, wrong dimensions, a user's own
+        error(...) call, a syntax error, ...). Calling lasterror() clears
+        it, so this also resets it for the next cell.
+        """
+        try:
+            resp = super().do_execute_direct(_CHECK_ERROR_CMD, True)
+        except Exception:  # noqa: BLE001 -- best-effort check, never fatal to the cell
+            return False
+        if not resp:
+            return False
+        match = _ERROR_FLAG_RE.search(resp.output)
+        return bool(match) and match.group(1) == 'string'
+
+    def _flush_pending_output(self):
+        pending, self._pending_chunk = self._pending_chunk, None
+        if pending is None:
+            return
+        if self._had_scilab_error():
+            # Error() wraps the whole message in RED...NORMAL, but Scilab's
+            # own terminal control codes (see _ANSI_ESCAPE_RE above) glued
+            # onto raw output include a reset (\x1b[0m) of their own --
+            # left in, it cancels the red partway through the message.
+            self.Error(_ANSI_ESCAPE_RE.sub('', pending).strip())
+        else:
+            super().Write(pending)
 
     def do_execute_direct(self, code, silent=False):
         if self._first:
             self._first = False
             self.handle_plot_settings()
             setup = self._setup.strip()
             self.do_execute_direct(setup, True)
-        resp = super(ScilabKernel, self).do_execute_direct(code, silent=silent)
+            # try/catch in the setup script above (getd(".")) can leave an
+            # error behind; clear it so it isn't mistaken for one of the
+            # user's own further down.
+            self._had_scilab_error()
+        resp = super().do_execute_direct(code, silent=silent)
+        if not silent:
+            self._flush_pending_output()
         if silent:
             return resp
         if self.plot_settings.get('backend', None) == 'inline':
@@ -198,24 +311,62 @@ def get_kernel_help_on(self, info, level=0, none_on_fail=False):
                 return None
             else:
                 return ""
-        self.do_execute_direct('help %s' % obj, True)
+        self.do_execute_direct(f'help {obj}', True)
 
     def do_shutdown(self, restart):
         self.wrapper.sendline('quit')
-        super(ScilabKernel, self).do_shutdown(restart)
+        super().do_shutdown(restart)
 
     def get_completions(self, info):
         """
         Get completions from kernel based on info dict.
         """
-        cmd = 'completion("%s")' % info['obj']
+        obj = info['obj']
+        # metakernel's do_complete() calls get_completions() even when the
+        # cursor sits right after a non-identifier character (e.g. "+",
+        # "-", "/") and there is no partial word to complete -- info['obj']
+        # is "" in that case. completion("") then matches every name
+        # Scilab knows, which is intentional: it's the same convention
+        # IPython follows (Tab on an empty prefix browses everything), and
+        # scilab-adv-cli's own interactive completion does the same thing.
+        #
+        # completion() displays its result on Scilab's own console, quoted
+        # (Scilab now shows string arrays as `"a"  "b"  ...`), which broke
+        # the parsing below; printf("%s\n", ...) instead prints each match
+        # bare, one per line. Only printf when there is at least one match:
+        # it errors out on an empty array. Both calls run in the same
+        # persistent Scilab session as the user's own code, so nothing here
+        # is assigned to a variable that could shadow one of theirs.
+        cmd = (
+            f'if ~isempty(completion("{obj}")) then '
+            f'printf("%s\\n",completion("{obj}")); end'
+        )
         output = self.do_execute_direct(cmd, True)
         if not output:
             return []
-        output = output.output.replace('!', '')
-        return [line.strip() for line in output.splitlines()
+        text = _ANSI_ESCAPE_RE.sub('', output.output)
+        return [line.strip() for line in text.splitlines()
                 if info['obj'] in line]
 
+    async def do_complete(self, code, cursor_pos):
+        content = await super().do_complete(code, cursor_pos)
+        if not content.get('matches'):
+            # metakernel's parser sets cursor_start to 0 (start of buffer)
+            # whenever there is no partial word at the cursor, regardless
+            # of whether any completions were actually found -- e.g. after
+            # "plot(1:10,", cursor_start ends up 0 with cursor_end at the
+            # cursor position, even though there is nothing to complete.
+            # With zero matches that wide range should never matter to a
+            # kernel-only completer, but JupyterLab also has its own
+            # generic word-from-document completer active alongside the
+            # kernel one, and a wide, kernel-reported range spanning real
+            # words ("plot", "1", "10") gives it something to offer -- an
+            # unwanted menu of words already on the line. Collapsing to a
+            # zero-width point at the cursor whenever we have nothing
+            # avoids handing it that range.
+            content['cursor_start'] = content['cursor_end']
+        return content
+
     def handle_plot_settings(self):
         """Handle the current plot settings"""
         settings = self.plot_settings
@@ -238,17 +389,17 @@ def handle_plot_settings(self):
             try:
                 width, height = settings['size'].split(',')
                 width, height = int(width), int(height)
-            except Exception as e:
-                self.Error('Error setting plot settings: %s' % e)
+            except (ValueError, AttributeError) as e:
+                self.Error(f'Error setting plot settings: {e}')
 
-        cmds.append('h.figure_size = [%s,%s];' % (width, height))
-        cmds.append('h.axes_size = [%s * 0.98, %s * 0.8];' % (width, height))
+        cmds.append(f'h.figure_size = [{width},{height}];')
+        cmds.append(f'h.axes_size = [{width} * 0.98, {height} * 0.8];')
 
         if settings['backend'] == 'inline':
             cmds.append('h.visible = "off";')
         else:
             cmds.append('h.visible = "on";')
-        super(ScilabKernel, self).do_execute_direct('\n'.join(cmds), True)
+        super().do_execute_direct('\n'.join(cmds), True)
 
     def make_figures(self, plot_dir=None):
         """Create figures for the current figures.
@@ -267,7 +418,7 @@ def make_figures(self, plot_dir=None):
         plot_format = self._plot_fmt.lower()
         make_figs = '_make_figures("%s", "%s");'
         make_figs = make_figs % (plot_dir, plot_format)
-        super(ScilabKernel, self).do_execute_direct(make_figs, True)
+        super().do_execute_direct(make_figs, True)
         return plot_dir
 
     def extract_figures(self, plot_dir):
@@ -291,7 +442,7 @@ def extract_figures(self, plot_dir):
                 if self.error_handler:
                     self.error_handler(e)
                 else:
-                    raise e
+                    raise
         return images
 
     def _handle_svg(self, filename):
@@ -305,14 +456,14 @@ def _handle_svg(self, filename):
         im = SVG(data=data)
         try:
             im.data = self._fix_svg_size(im.data)
-        except Exception:
-            pass
+        except (ValueError, ExpatError) as e:
+            self.log.debug(f'Could not resize SVG (unexpected shape from GnuPlot?): {e}')
         try:
             settings = self.plot_settings
             if settings['antialiasing']:
                 im.data = self._fix_svg_antialiasing(im.data)
-        except Exception:
-            pass
+        except (ValueError, ExpatError) as e:
+            self.log.debug(f'Could not adjust SVG antialiasing (unexpected shape from GnuPlot?): {e}')
         return im
 
     def _fix_svg_size(self, data):
@@ -340,8 +491,8 @@ def _fix_svg_size(self, data):
                 width = width * settings['height'] / height
             height = settings['height']
 
-        svg.setAttribute('width', '%dpx' % width)
-        svg.setAttribute('height', '%dpx' % height)
+        svg.setAttribute('width', f'{int(width)}px')
+        svg.setAttribute('height', f'{int(height)}px')
         return svg.toxml()
 
     def _fix_svg_antialiasing(self, data):