Gridarta Editor
TokenMarker.java
Go to the documentation of this file.
1 /*
2  * TokenMarker.java - Generic token marker
3  * Copyright (C) 1998, 1999 Slava Pestov
4  * Copyright (C) 2000-2023 The Gridarta Developers.
5  *
6  * You may use and modify this package for any purpose. Redistribution is
7  * permitted, in both source and binary form, provided that this notice
8  * remains intact in all source distributions of this package.
9  */
10 
11 package net.sf.gridarta.textedit.textarea.tokenmarker;
12 
13 import java.util.LinkedList;
14 import java.util.List;
15 import javax.swing.text.Segment;
17 import org.jetbrains.annotations.NotNull;
18 import org.jetbrains.annotations.Nullable;
19 
32 public abstract class TokenMarker {
33 
37  @NotNull
38  private final List<Token> tokens = new LinkedList<>();
39 
45  @NotNull
46  private LineInfo @Nullable [] lineInfo;
47 
52  private int length;
53 
57  private int lastLine;
58 
62  private boolean nextLineRequested;
63 
71  @NotNull
72  public List<Token> markTokens(@NotNull final Segment line, final int lineIndex) {
73  if (lineIndex >= length) {
74  throw new IllegalArgumentException("Tokenizing invalid line: " + lineIndex);
75  }
76 
77  tokens.clear();
78 
79  final LineInfo info = lineInfo[lineIndex];
80  final LineInfo prev = lineIndex == 0 ? null : lineInfo[lineIndex - 1];
81 
82  final byte oldToken = info.getToken();
83  final byte token = markTokensImpl(prev == null ? Token.NULL : prev.getToken(), line);
84 
85  info.setToken(token);
86 
87  /*
88  * This is a foul hack. It stops nextLineRequested
89  * from being cleared if the same line is marked twice.
90  *
91  * Why is this necessary? It's all JEditTextArea's fault.
92  * When something is inserted into the text, firing a
93  * document event, the insertUpdate() method shifts the
94  * caret (if necessary) by the amount inserted.
95  *
96  * All caret movement is handled by the select() method,
97  * which eventually pipes the new position to scrollTo()
98  * and calls repaint().
99  *
100  * Note that at this point in time, the new line hasn't
101  * yet been painted; the caret is moved first.
102  *
103  * scrollTo() calls offsetToX(), which tokenizes the line
104  * unless it is being called on the last line painted
105  * (in which case it uses the text area's painter cached
106  * token list). What scrollTo() does next is irrelevant.
107  *
108  * After scrollTo() has done it's job, repaint() is
109  * called, and eventually we end up in paintLine(), whose
110  * job is to paint the changed line. It, too, calls
111  * markTokens().
112  *
113  * The problem was that if the line started a multi.line
114  * token, the first markTokens() (done in offsetToX())
115  * would set nextLineRequested (because the line end
116  * token had changed) but the second would clear it
117  * (because the line was the same that time) and therefore
118  * paintLine() would never know that it needed to repaint
119  * subsequent lines.
120  *
121  * This bug took me ages to track down, that's why I wrote
122  * all the relevant info down so that others wouldn't
123  * duplicate it.
124  */
125  if (!(lastLine == lineIndex && nextLineRequested)) {
126  nextLineRequested = oldToken != token;
127  }
128 
129  lastLine = lineIndex;
130 
131  return tokens;
132  }
133 
146  protected abstract byte markTokensImpl(byte token, @NotNull Segment line);
147 
154  public void insertLines(final int index, final int lines) {
155  if (lines <= 0) {
156  return;
157  }
158  length += lines;
160  final int len = index + lines;
161  System.arraycopy(lineInfo, index, lineInfo, len, lineInfo.length - len);
162 
163  for (int i = index + lines - 1; i >= index; i--) {
164  lineInfo[i] = new LineInfo();
165  }
166  }
167 
174  public void deleteLines(final int index, final int lines) {
175  if (lines <= 0) {
176  return;
177  }
178  length -= lines;
179  final int len = index + lines;
180  System.arraycopy(lineInfo, len, lineInfo, index, lineInfo.length - len);
181  }
182 
188  public boolean isNextLineRequested() {
189  return nextLineRequested;
190  }
191 
196  protected TokenMarker() {
197  lastLine = -1;
198  }
199 
210  private void ensureCapacity(final int index) {
211  if (lineInfo == null) {
212  lineInfo = new LineInfo[index + 1];
213  } else if (lineInfo.length <= index) {
214  final LineInfo[] lineInfoN = new LineInfo[(index + 1) * 2];
215  System.arraycopy(lineInfo, 0, lineInfoN, 0, lineInfo.length);
216  lineInfo = lineInfoN;
217  }
218  }
219 
225  protected void addToken(final int length, final byte id) {
226  if (id >= Token.INTERNAL_FIRST && id <= Token.INTERNAL_LAST) {
227  throw new InternalError("Invalid id: " + id);
228  }
229 
230  if (length == 0) {
231  return;
232  }
233 
234  tokens.add(new Token(length, id));
235  }
236 
237 }
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.tokens
final List< Token > tokens
The collected tokens.
Definition: TokenMarker.java:38
net.sf.gridarta.textedit.textarea.tokenmarker.LineInfo.getToken
byte getToken()
Definition: LineInfo.java:55
net.sf.gridarta.textedit.textarea.Token.INTERNAL_FIRST
static final byte INTERNAL_FIRST
The first id that can be used for internal state in a token marker.
Definition: Token.java:94
net.sf.gridarta.textedit.textarea
This package contains the other part of the script editor.
net.sf.gridarta
Base package of all Gridarta classes.
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.TokenMarker
TokenMarker()
Creates a new.
Definition: TokenMarker.java:196
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.addToken
void addToken(final int length, final byte id)
Adds a token to the token list.
Definition: TokenMarker.java:225
net.sf.gridarta.textedit.textarea.Token.NULL
static final byte NULL
Normal text token id.
Definition: Token.java:26
net.sf
net.sf.gridarta.textedit
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.lastLine
int lastLine
The last tokenized line.
Definition: TokenMarker.java:57
net.sf.gridarta.textedit.textarea.tokenmarker.LineInfo.setToken
void setToken(final byte token)
Definition: LineInfo.java:59
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.isNextLineRequested
boolean isNextLineRequested()
Returns true if the next line should be repainted.
Definition: TokenMarker.java:188
net.sf.gridarta.textedit.textarea.Token
A linked list of tokens.
Definition: Token.java:21
net.sf.gridarta.textedit.textarea.Token.INTERNAL_LAST
static final byte INTERNAL_LAST
The last id that can be used for internal state in a token marker.
Definition: Token.java:99
net
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.insertLines
void insertLines(final int index, final int lines)
Informs the token marker that lines have been inserted into the document.
Definition: TokenMarker.java:154
net.sf.gridarta.textedit.textarea.tokenmarker.LineInfo
Stores information about tokenized lines.
Definition: LineInfo.java:27
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.ensureCapacity
void ensureCapacity(final int index)
Ensures that the.
Definition: TokenMarker.java:210
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.lineInfo
LineInfo[] lineInfo
An array for storing information about lines.
Definition: TokenMarker.java:46
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.nextLineRequested
boolean nextLineRequested
True if the next line should be painted.
Definition: TokenMarker.java:62
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker
A token marker that splits lines of text into tokens.
Definition: TokenMarker.java:32
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.markTokensImpl
abstract byte markTokensImpl(byte token, @NotNull Segment line)
An abstract method that splits a line up into tokens.
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.length
int length
The number of lines in the model being tokenized.
Definition: TokenMarker.java:52
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.deleteLines
void deleteLines(final int index, final int lines)
Informs the token marker that line have been deleted from the document.
Definition: TokenMarker.java:174
net.sf.gridarta.textedit.textarea.tokenmarker.TokenMarker.markTokens
List< Token > markTokens(@NotNull final Segment line, final int lineIndex)
A wrapper for the lower-level.
Definition: TokenMarker.java:72