How do you add folding to QsciLexerCustom subclass?

Question

Consider this snippet:

import sys
import textwrap
import re

from PyQt5.Qt import *  # noqa

from PyQt5.Qsci import QsciScintilla
from PyQt5.Qsci import QsciLexerCustom

from lark import Lark, inline_args, Transformer


class LexerJson(QsciLexerCustom):

    def __init__(self, parent=None):
        super().__init__(parent)
        self.create_grammar()
        self.create_styles()

    def create_styles(self):
        deeppink = QColor(249, 38, 114)
        khaki = QColor(230, 219, 116)
        mediumpurple = QColor(174, 129, 255)
        mediumturquoise = QColor(81, 217, 205)
        yellowgreen = QColor(166, 226, 46)
        lightcyan = QColor(213, 248, 232)
        darkslategrey = QColor(39, 40, 34)

        styles = {
            0: mediumturquoise,
            1: mediumpurple,
            2: yellowgreen,
            3: deeppink,
            4: khaki,
            5: lightcyan
        }

        for style, color in styles.items():
            self.setColor(color, style)
            self.setPaper(darkslategrey, style)
            self.setFont(self.parent().font(), style)

        self.token_styles = {
            "__COLON": 5,
            "__COMMA": 5,
            "__FALSE1": 0,
            "__LBRACE": 5,
            "__LSQB": 5,
            "__NULL2": 0,
            "__RBRACE": 5,
            "__RSQB": 5,
            "__TRUE0": 0,
            "ESCAPED_STRING": 4,
            "SIGNED_NUMBER": 1,
        }

    def create_grammar(self):
        grammar = '''
            ?start: value
            ?value: object
                  | array
                  | string
                  | SIGNED_NUMBER      -> number
                  | "true"             -> true
                  | "false"            -> false
                  | "null"             -> null
            array  : "[" [value ("," value)*] "]"
            object : "{" [pair ("," pair)*] "}"
            pair   : string ":" value
            string : ESCAPED_STRING
            %import common.ESCAPED_STRING
            %import common.SIGNED_NUMBER
            %import common.WS
            %ignore WS
        '''

        class TreeToJson(Transformer):
            @inline_args
            def string(self, s):
                return s[1:-1].replace('\\"', '"')

            array = list
            pair = tuple
            object = dict
            number = inline_args(float)

            def null(self, _): return None

            def true(self, _): return True

            def false(self, _): return False

        self.lark = Lark(grammar, parser='lalr', transformer=TreeToJson())
        # All tokens: print([t.name for t in self.lark.parser.lexer.tokens])

    def defaultPaper(self, style):
        return QColor(39, 40, 34)

    def language(self):
        return "Json"

    def description(self, style):
        return {v: k for k, v in self.token_styles.items()}.get(style, "")

    def styleText(self, start, end):
        self.startStyling(start)
        text = self.parent().text()[start:end]
        last_pos = 0

        try:
            for token in self.lark.lex(text):
                ws_len = token.pos_in_stream - last_pos
                if ws_len:
                    self.setStyling(ws_len, 0)    # whitespace

                token_len = len(bytearray(token, "utf-8"))
                self.setStyling(
                    token_len, self.token_styles.get(token.type, 0))

                last_pos = token.pos_in_stream + token_len
        except Exception as e:
            print(e)


class EditorAll(QsciScintilla):

    def __init__(self, parent=None):
        super().__init__(parent)

        # Set font defaults
        font = QFont()
        font.setFamily('Consolas')
        font.setFixedPitch(True)
        font.setPointSize(8)
        font.setBold(True)
        self.setFont(font)

        # Set margin defaults
        fontmetrics = QFontMetrics(font)
        self.setMarginsFont(font)
        self.setMarginWidth(0, fontmetrics.width("000") + 6)
        self.setMarginLineNumbers(0, True)
        self.setMarginsForegroundColor(QColor(128, 128, 128))
        self.setMarginsBackgroundColor(QColor(39, 40, 34))
        self.setMarginType(1, self.SymbolMargin)
        self.setMarginWidth(1, 12)

        # Set indentation defaults
        self.setIndentationsUseTabs(False)
        self.setIndentationWidth(4)
        self.setBackspaceUnindents(True)
        self.setIndentationGuides(True)

        # Set folding defaults (http://www.scintilla.org/ScintillaDoc.html#Folding)
        self.setFolding(QsciScintilla.CircledFoldStyle)

        # Set caret defaults
        self.setCaretForegroundColor(QColor(247, 247, 241))
        self.setCaretWidth(2)

        # Set selection color defaults
        self.setSelectionBackgroundColor(QColor(61, 61, 52))
        self.resetSelectionForegroundColor()

        # Set multiselection defaults
        self.SendScintilla(QsciScintilla.SCI_SETMULTIPLESELECTION, True)
        self.SendScintilla(QsciScintilla.SCI_SETMULTIPASTE, 1)
        self.SendScintilla(
            QsciScintilla.SCI_SETADDITIONALSELECTIONTYPING, True)

        lexer = LexerJson(self)
        self.setLexer(lexer)


def main():
    app = QApplication(sys.argv)
    ex = EditorAll()
    ex.setWindowTitle(__file__)
    ex.setText(textwrap.dedent("""\
        {
            "_id": "5b05ffcbcf8e597939b3f5ca",
            "about": "Excepteur consequat commodo esse voluptate aute aliquip ad sint deserunt commodo eiusmod irure. Sint aliquip sit magna duis eu est culpa aliqua excepteur ut tempor nulla. Aliqua ex pariatur id labore sit. Quis sit ex aliqua veniam exercitation laboris anim adipisicing. Lorem nisi reprehenderit ullamco labore qui sit ut aliqua tempor consequat pariatur proident.",
            "address": "665 Malbone Street, Thornport, Louisiana, 243",
            "age": 23,
            "balance": "$3,216.91",
            "company": "BULLJUICE",
            "email": "[email protected]",
            "eyeColor": "brown",
            "gender": "female",
            "guid": "d3a6d865-0f64-4042-8a78-4f53de9b0707",
            "index": 0,
            "isActive": false,
            "isActive2": true,
            "latitude": -18.660714,
            "longitude": -85.378048,
            "name": "Elise Kelley",
            "phone": "[phone]",
            "picture": "http://placehold.it/32x32",
            "registered": "2017-09-30T03:47:40 -02:00",
            "tags": [
                "et",
                "nostrud",
                "in",
                "fugiat",
                "incididunt",
                "labore",
                "nostrud"
            ]
        }\
    """))
    ex.resize(800, 600)
    ex.show()
    sys.exit(app.exec_())


if __name__ == "__main__":
    main()

To run the above mcve you'll just need to run pip install lark-parser PyQt5 QScintilla

I'm trying to figure out how to modify LexerJson so the symbols [ ] { } will support folding. When using an existing class such as qscilexercpp.cpp the folding behaviour is given to you just for free, for instance, you'd just do something like:

# http://www.scintilla.org/ScintillaDoc.html#Folding
self.setFolding(QsciScintilla.BoxedTreeFoldStyle)

lexer = QsciLexerCPP()
lexer.setFoldAtElse(True)
lexer.setFoldComments(True)
lexer.setFoldCompact(False)
lexer.setFoldPreprocessor(True)
self.setLexer(lexer)

And folding would work just for free... but when using a custom lexer like I'm doing in the posted mcve I guess you got to implement that behaviour yourself, unfortunately I don't know how to do it.

So, that's basically the question, how do you implement folding on a QsciLexerCustom subclass?

Firing

any feedback on the answer? It is a working example and the bounty is about to expire – Marabou

@TarunLalwani Well, If in few hours no better answer comes I guess I'll bounty and validate your answer but you should already know at this point why your answer hasn't received more upvotes considering this is a good question. It gives enough information about the general question but it hasn't addressed the mcve at all, so... Btw, at first I thought I wasn't bounty/validate it but then I considered your answer already provides enough info to work out my real word example (glsl lexer) so... I'll wait few hours though in case someone else give it a shot – Firing

Agreed, the thing is that it took a decent amount of time to even find any information on this. After digging a lot I found one working example. That is the reason I didn't want to spend more time and create a workable solution for your JSON problem. So left the answer more as pointer to help you continue the work – Marabou

Answer 1

I can't fix your lexer code, but I can give you a working example for the same

import sys
from PyQt5.Qt import *
from PyQt5.Qsci import QsciScintilla, QsciLexerCPP
from PyQt5.Qsci import QsciLexerCustom

if sys.hexversion < 0x020600F0:
     print('python 2.6 or greater is required by this program')
     sys.exit(1)

_sample = """
# Sample config file

this is a junk line

[FirstItem]
Width=100
Height=200
Colour=orange
Info=this is some
     multiline
     text

[SecondItem]
Width=200
Height=300
Colour=green
Info=
     this is some
     multiline
     text
"""


class MainWindow(QMainWindow):
     def __init__(self):
         QMainWindow.__init__(self)
         self.setWindowTitle('Custom Lexer For Config Files')
         self.setGeometry(50, 200, 400, 400)
         self.editor = QsciScintilla(self)
         self.editor.setUtf8(True)
         self.editor.setMarginWidth(2, 15)
         self.editor.setFolding(True)
         self.setCentralWidget(self.editor)
         self.lexer = ConfigLexer(self.editor)
         self.editor.setLexer(self.lexer)
         self.editor.setText(_sample)


class ConfigLexer(QsciLexerCustom):
     def __init__(self, parent):
         QsciLexerCustom.__init__(self, parent)
         self._styles = {
             0: 'Default',
             1: 'Comment',
             2: 'Section',
             3: 'Key',
             4: 'Assignment',
             5: 'Value',
             }
         for key,value in self._styles.items():
             setattr(self, value, key)
         self._foldcompact = True

     def foldCompact(self):
         return self._foldcompact

     def setFoldCompact(self, enable):
         self._foldcompact = bool(enable)

     def language(self):
         return 'Config Files'

     def description(self, style):
         return self._styles.get(style, '')

     def defaultColor(self, style):
         if style == self.Default:
             return QColor('#000000')
         elif style == self.Comment:
             return QColor('#A0A0A0')
         elif style == self.Section:
             return QColor('#CC6600')
         elif style == self.Key:
             return QColor('#0000CC')
         elif style == self.Assignment:
             return QColor('#CC0000')
         elif style == self.Value:
             return QColor('#00CC00')
         return QsciLexerCustom.defaultColor(self, style)

     def defaultPaper(self, style):
         if style == self.Section:
             return QColor('#FFEECC')
         return QsciLexerCustom.defaultPaper(self, style)

     def defaultEolFill(self, style):
         if style == self.Section:
             return True
         return QsciLexerCustom.defaultEolFill(self, style)

     def defaultFont(self, style):
         if style == self.Comment:
             if sys.platform in ('win32', 'cygwin'):
                 return QFont('Comic Sans MS', 9)
             return QFont('Bitstream Vera Serif', 9)
         return QsciLexerCustom.defaultFont(self, style)

     def styleText(self, start, end):
         editor = self.editor()
         if editor is None:
             return

         SCI = editor.SendScintilla
         GETFOLDLEVEL = QsciScintilla.SCI_GETFOLDLEVEL
         SETFOLDLEVEL = QsciScintilla.SCI_SETFOLDLEVEL
         HEADERFLAG = QsciScintilla.SC_FOLDLEVELHEADERFLAG
         LEVELBASE = QsciScintilla.SC_FOLDLEVELBASE
         NUMBERMASK = QsciScintilla.SC_FOLDLEVELNUMBERMASK
         WHITEFLAG = QsciScintilla.SC_FOLDLEVELWHITEFLAG
         set_style = self.setStyling

         source = ''
         if end > editor.length():
             end = editor.length()
         if end > start:
             source = bytearray(end - start)
             SCI(QsciScintilla.SCI_GETTEXTRANGE, start, end, source)
         if not source:
             return

         compact = self.foldCompact()

         index = SCI(QsciScintilla.SCI_LINEFROMPOSITION, start)
         if index > 0:
             pos = SCI(QsciScintilla.SCI_GETLINEENDPOSITION, index - 1)
             state = SCI(QsciScintilla.SCI_GETSTYLEAT, pos)
         else:
             state = self.Default

         self.startStyling(start, 0x1f)

         for line in source.splitlines(True):
             length = len(line)
             if length == 1:
                 whitespace = compact
                 state = self.Default
             else:
                 whitespace = False
                 firstchar = chr(line[0])
                 if firstchar in '#;':
                     state = self.Comment
                 elif firstchar == '[':
                     state = self.Section
                 elif firstchar in ' \t':
                     if state == self.Value or state == self.Assignment:
                         state = self.Value
                     else:
                         whitespace = compact and line.isspace()
                         state = self.Default
                 else:
                     pos = line.find(b'=')
                     if pos < 0:
                         pos = line.find(b':')
                     else:
                         tmp = line.find(b':', 0, pos)
                         if tmp >= 0:
                             pos = tmp
                     if pos > 0:
                         set_style(pos, self.Key)
                         set_style(1, self.Assignment)
                         length = length - pos - 1
                         state = self.Value
                     else:
                         state = self.Default
             set_style(length, state)

             if state == self.Section:
                 level = LEVELBASE | HEADERFLAG
             elif index > 0:
                 lastlevel = SCI(GETFOLDLEVEL, index - 1)
                 if lastlevel & HEADERFLAG:
                     level = LEVELBASE + 1
                 else:
                     level = lastlevel & NUMBERMASK
             else:
                 level = LEVELBASE

             if whitespace:
                 level |= WHITEFLAG
             if level != SCI(GETFOLDLEVEL, index):
                 SCI(SETFOLDLEVEL, index, level)

             index += 1

         if index > 0:
             lastlevel = SCI(GETFOLDLEVEL, index - 1)
             if lastlevel & HEADERFLAG:
                 level = LEVELBASE + 1
             else:
                 level = lastlevel & NUMBERMASK
         else:
             level = LEVELBASE

         lastlevel = SCI(GETFOLDLEVEL, index)
         SCI(SETFOLDLEVEL, index, level | lastlevel & ~NUMBERMASK)


if __name__ == "__main__":
     app = QApplication(sys.argv)
     win = MainWindow()
     win.show()
     sys.exit(app.exec_())

The key stuff happens here

             if state == self.Section:
                 level = LEVELBASE | HEADERFLAG
             elif index > 0:
                 lastlevel = SCI(GETFOLDLEVEL, index - 1)
                 if lastlevel & HEADERFLAG:
                     level = LEVELBASE + 1
                 else:
                     level = lastlevel & NUMBERMASK
             else:
                 level = LEVELBASE

             if whitespace:
                 level |= WHITEFLAG
             if level != SCI(GETFOLDLEVEL, index):
                 SCI(SETFOLDLEVEL, index, level)

             index += 1

         if index > 0:
             lastlevel = SCI(GETFOLDLEVEL, index - 1)
             if lastlevel & HEADERFLAG:
                 level = LEVELBASE + 1
             else:
                 level = lastlevel & NUMBERMASK
         else:
             level = LEVELBASE

         lastlevel = SCI(GETFOLDLEVEL, index)
         SCI(SETFOLDLEVEL, index, level | lastlevel & ~NUMBERMASK)

PS: Credits to https://github.com/pingf/toys/blob/f808e7c4ed1a76db4800c8e1ee6d163242df52cc/src/201403/customLexer2.py

Marabou

Ok, considering you've spent a considerable ammount of time finding the example, also the +1 . Thanks for the answer btw – Firing