Gridarta Editor
LuaTokenMarker.java
Go to the documentation of this file.
1 /*
2  * LuaTokenMarker.java - Lua token marker
3  *
4  * Based on:
5  *
6  * PythonTokenMarker.java - Python token marker
7  * Copyright (C) 1999 Jonathan Revusky
8  * Copyright (C) 1998, 1999 Slava Pestov
9  * Copyright (C) 2000-2023 The Gridarta Developers.
10  *
11  * You may use and modify this package for any purpose. Redistribution is
12  * permitted, in both source and binary form, provided that this notice
13  * remains intact in all source distributions of this package.
14  */
15 
16 package net.sf.gridarta.textedit.textarea.tokenmarker;
17 
18 import javax.swing.text.Segment;
21 import org.jetbrains.annotations.NotNull;
22 import org.jetbrains.annotations.Nullable;
23 
27 public class LuaTokenMarker extends TokenMarker {
28 
29  private static final byte SINGLE_QUOTE = Token.INTERNAL_FIRST;
30 
31  private static final byte DOUBLE_QUOTE = Token.INTERNAL_FIRST + 1;
32 
33  private static final byte MULTI_COMMENT = Token.INTERNAL_FIRST + 2;
34 
35  private static final byte MULTI_LITERAL = Token.INTERNAL_LAST;
36 
37  @Nullable
38  private static KeywordMap luaKeywords;
39 
40  @NotNull
41  private final KeywordMap keywords;
42 
43  private int lastOffset;
44 
45  private int lastKeyword;
46 
47  public LuaTokenMarker() {
49  }
50 
51  @Override
52  public byte markTokensImpl(final byte token, @NotNull final Segment line) {
53  final char[] array = line.array;
54  final int offset = line.offset;
55  lastOffset = offset;
56  lastKeyword = offset;
57  final int length = line.count + offset;
58  boolean backslash = false;
59 
60  byte currentToken = token;
61 loop:
62  for (int i = offset; i < length; i++) {
63  final int i1 = i + 1;
64 
65  final char c = array[i];
66  if (c == '\\') {
67  backslash = !backslash;
68  continue;
69  }
70 
71  switch (currentToken) {
72  case Token.NULL:
73  if (backslash) {
74  backslash = false;
75  }
76  switch (c) {
77  case '-':
78  if (i < length && array[i1] == '-') {
79  doKeyword(line, i);
80  addToken(i - lastOffset, currentToken);
81  if (i < length - 2 && SyntaxUtilities.regionMatches(false, line, i + 2, "[[")) {
82  currentToken = MULTI_COMMENT;
83  lastOffset = i;
84  lastKeyword = i;
85  i += 3;
86  } else {
90  break loop;
91  }
92  }
93  break;
94  case '[':
95  if (i < length && array[i1] == '[') {
96  doKeyword(line, i);
97  addToken(i - lastOffset, currentToken);
98  currentToken = MULTI_LITERAL;
99  lastOffset = i;
100  lastKeyword = i;
101  i++;
102  }
103  break;
104  case '"':
105  doKeyword(line, i);
106  addToken(i - lastOffset, currentToken);
107  currentToken = DOUBLE_QUOTE;
108  lastOffset = i;
109  lastKeyword = i;
110  break;
111  case '\'':
112  doKeyword(line, i);
113  addToken(i - lastOffset, currentToken);
114  currentToken = SINGLE_QUOTE;
115  lastOffset = i;
116  lastKeyword = i;
117  break;
118  default:
119  if (!Character.isLetterOrDigit(c) && c != '_') {
120  doKeyword(line, i);
121  }
122  break;
123  }
124  break;
125  case SINGLE_QUOTE:
126  if (backslash) {
127  backslash = false;
128  } else if (c == '\'') {
130  currentToken = Token.NULL;
131  lastOffset = i1;
132  lastKeyword = i1;
133  }
134  break;
135  case DOUBLE_QUOTE:
136  if (backslash) {
137  backslash = false;
138  } else if (c == '"') {
140  currentToken = Token.NULL;
141  lastOffset = i1;
142  lastKeyword = i1;
143  }
144  break;
145  case MULTI_COMMENT:
146  if (backslash) {
147  backslash = false;
148  }
149  if (SyntaxUtilities.regionMatches(false, line, i, "]]")) {
150  i += 2;
152  currentToken = Token.NULL;
153  lastOffset = i;
154  lastKeyword = i;
155  }
156  break;
157  case MULTI_LITERAL:
158  if (backslash) {
159  backslash = false;
160  }
161  if (SyntaxUtilities.regionMatches(false, line, i, "]]")) {
162  i += 2;
164  currentToken = Token.NULL;
165  lastOffset = i;
166  lastKeyword = i;
167  }
168  break;
169  default:
170  throw new InternalError("Invalid state: " + currentToken);
171  }
172  }
173 
174  switch (currentToken) {
175  case SINGLE_QUOTE:
176  case DOUBLE_QUOTE:
177  case MULTI_LITERAL:
179  break;
180  case MULTI_COMMENT:
182  break;
183  case Token.NULL:
184  doKeyword(line, length);
185  default:
186  addToken(length - lastOffset, currentToken);
187  break;
188  }
189 
190  return currentToken;
191  }
192 
193  @NotNull
194  private static KeywordMap getLuaKeywords() {
195  if (luaKeywords == null) {
196  luaKeywords = new KeywordMap(false);
197  luaKeywords.add("break", Token.KEYWORD1);
198  luaKeywords.add("do", Token.KEYWORD1);
199  luaKeywords.add("else", Token.KEYWORD1);
200  luaKeywords.add("elseif", Token.KEYWORD1);
201  luaKeywords.add("end", Token.KEYWORD1);
202  luaKeywords.add("for", Token.KEYWORD1);
203  luaKeywords.add("function", Token.KEYWORD1);
204  luaKeywords.add("if", Token.KEYWORD1);
205  luaKeywords.add("in", Token.KEYWORD1);
206  luaKeywords.add("local", Token.KEYWORD1);
207  luaKeywords.add("repeat", Token.KEYWORD1);
208  luaKeywords.add("require", Token.KEYWORD1);
209  luaKeywords.add("return", Token.KEYWORD1);
210  luaKeywords.add("then", Token.KEYWORD1);
211  luaKeywords.add("until", Token.KEYWORD1);
212  luaKeywords.add("while", Token.KEYWORD1);
213  luaKeywords.add("false", Token.LITERAL2);
214  luaKeywords.add("nil", Token.LITERAL2);
215  luaKeywords.add("true", Token.LITERAL2);
216  luaKeywords.add("and", Token.OPERATOR);
217  luaKeywords.add("not", Token.OPERATOR);
218  luaKeywords.add("or", Token.OPERATOR);
219  }
220  return luaKeywords;
221  }
222 
223  private void doKeyword(@NotNull final Segment line, final int i) {
224  final int len = i - lastKeyword;
225  final byte id = keywords.lookup(line, lastKeyword, len);
226  if (id != Token.NULL) {
227  if (lastKeyword != lastOffset) {
229  }
230  addToken(len, id);
231  lastOffset = i;
232  }
233  lastKeyword = i + 1;
234  }
235 
236 }
net.sf.gridarta.textedit.textarea.Token.INTERNAL_FIRST
static final byte INTERNAL_FIRST
The first id that can be used for internal state in a token marker.
Definition: Token.java:94
net.sf.gridarta.textedit.textarea
This package contains the other part of the script editor.
net.sf.gridarta
Base package of all Gridarta classes.
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.addToken
void addToken(final int length, final byte id)
Adds a token to the token list.
Definition: TokenMarker.java:225
net.sf.gridarta.textedit.textarea.Token.NULL
static final byte NULL
Normal text token id.
Definition: Token.java:26
net.sf
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker
Lua token marker.
Definition: LuaTokenMarker.java:27
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.DOUBLE_QUOTE
static final byte DOUBLE_QUOTE
Definition: LuaTokenMarker.java:31
net.sf.gridarta.textedit.textarea.tokenmarker.KeywordMap.lookup
byte lookup(final Segment text, final int offset, final int length)
Looks up a key.
Definition: KeywordMap.java:63
net.sf.gridarta.textedit
net.sf.gridarta.textedit.textarea.SyntaxUtilities.regionMatches
static boolean regionMatches(final boolean ignoreCase, final Segment text, final int offset, @NotNull final CharSequence match)
Checks if a sub-region of a.
Definition: SyntaxUtilities.java:43
net.sf.gridarta.textedit.textarea.tokenmarker.KeywordMap.add
void add(final CharSequence keyword, final byte id)
Adds a key-value mapping.
Definition: KeywordMap.java:79
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.markTokensImpl
byte markTokensImpl(final byte token, @NotNull final Segment line)
An abstract method that splits a line up into tokens.
Definition: LuaTokenMarker.java:52
net.sf.gridarta.textedit.textarea.Token.LITERAL2
static final byte LITERAL2
Literal 2 token id.
Definition: Token.java:48
net.sf.gridarta.textedit.textarea.Token
A linked list of tokens.
Definition: Token.java:21
net.sf.gridarta.textedit.textarea.Token.INTERNAL_LAST
static final byte INTERNAL_LAST
The last id that can be used for internal state in a token marker.
Definition: Token.java:99
net.sf.gridarta.textedit.textarea.Token.OPERATOR
static final byte OPERATOR
Operator token id.
Definition: Token.java:78
net
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.lastOffset
int lastOffset
Definition: LuaTokenMarker.java:43
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.keywords
final KeywordMap keywords
Definition: LuaTokenMarker.java:41
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.doKeyword
void doKeyword(@NotNull final Segment line, final int i)
Definition: LuaTokenMarker.java:223
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.lastKeyword
int lastKeyword
Definition: LuaTokenMarker.java:45
net.sf.gridarta.textedit.textarea.Token.KEYWORD1
static final byte KEYWORD1
Keyword 1 token id.
Definition: Token.java:60
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.MULTI_LITERAL
static final byte MULTI_LITERAL
Definition: LuaTokenMarker.java:35
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.SINGLE_QUOTE
static final byte SINGLE_QUOTE
Definition: LuaTokenMarker.java:29
net.sf.gridarta.textedit.textarea.tokenmarker.KeywordMap
Maps (parts of) Segments to.
Definition: KeywordMap.java:35
net.sf.gridarta.textedit.textarea.SyntaxUtilities
Class with several utility functions used by jEdit's syntax colorizing subsystem.
Definition: SyntaxUtilities.java:27
net.sf.gridarta.textedit.textarea.Token.LITERAL1
static final byte LITERAL1
Literal 1 token id.
Definition: Token.java:42
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.luaKeywords
static KeywordMap luaKeywords
Definition: LuaTokenMarker.java:38
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.MULTI_COMMENT
static final byte MULTI_COMMENT
Definition: LuaTokenMarker.java:33
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.LuaTokenMarker
LuaTokenMarker()
Definition: LuaTokenMarker.java:47
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker
A token marker that splits lines of text into tokens.
Definition: TokenMarker.java:32
net.sf.gridarta.textedit.textarea.Token.COMMENT1
static final byte COMMENT1
Comment 1 token id.
Definition: Token.java:31
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.length
int length
The number of lines in the model being tokenized.
Definition: TokenMarker.java:52
net.sf.gridarta.textedit.textarea.tokenmarker.LuaTokenMarker.getLuaKeywords
static KeywordMap getLuaKeywords()
Definition: LuaTokenMarker.java:194