ExpressionParser: Add support for /* */ style comments.

Source/Core/DolphinQt/Config/Mapping/IOWindow.cpp

@@ -44,14 +44,6 @@ QTextCharFormat GetSpecialCharFormat()
   return format;
 }
 
-QTextCharFormat GetOperatorCharFormat()
-{
-  QTextCharFormat format;
-  format.setFontWeight(QFont::Weight::Bold);
-  format.setForeground(QBrush{Qt::darkBlue});
-  return format;
-}
-
 QTextCharFormat GetLiteralCharFormat()
 {
   QTextCharFormat format;
@@ -77,14 +69,21 @@ QTextCharFormat GetControlCharFormat()
 QTextCharFormat GetVariableCharFormat()
 {
   QTextCharFormat format;
-  format.setForeground(QBrush{Qt::magenta});
+  format.setForeground(QBrush{Qt::darkYellow});
   return format;
 }
 
 QTextCharFormat GetBarewordCharFormat()
 {
   QTextCharFormat format;
-  format.setForeground(QBrush{Qt::darkCyan});
+  format.setForeground(QBrush{Qt::darkBlue});
+  return format;
+}
+
+QTextCharFormat GetCommentCharFormat()
+{
+  QTextCharFormat format;
+  format.setForeground(QBrush{Qt::darkGray});
   return format;
 }
 }  // namespace
@@ -95,16 +94,35 @@ ControlExpressionSyntaxHighlighter::ControlExpressionSyntaxHighlighter(QTextDocu
 {
 }
 
-void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
+void ControlExpressionSyntaxHighlighter::highlightBlock(const QString&)
 {
   // TODO: This is going to result in improper highlighting with non-ascii characters:
-  ciface::ExpressionParser::Lexer lexer(text.toStdString());
+  ciface::ExpressionParser::Lexer lexer(document()->toPlainText().toStdString());
 
   std::vector<ciface::ExpressionParser::Token> tokens;
   const auto tokenize_status = lexer.Tokenize(tokens);
 
   using ciface::ExpressionParser::TokenType;
 
+  const auto set_block_format = [this](int start, int count, const QTextCharFormat& format) {
+    if (start + count <= currentBlock().position() ||
+        start >= currentBlock().position() + currentBlock().length())
+    {
+      // This range is not within the current block.
+      return;
+    }
+
+    int block_start = start - currentBlock().position();
+
+    if (block_start < 0)
+    {
+      count += block_start;
+      block_start = 0;
+    }
+
+    setFormat(block_start, count, format);
+  };
+
   for (auto& token : tokens)
   {
     std::optional<QTextCharFormat> char_format;
@@ -131,22 +149,27 @@ void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
     case TokenType::TOK_VARIABLE:
       char_format = GetVariableCharFormat();
       break;
+    case TokenType::TOK_COMMENT:
+      char_format = GetCommentCharFormat();
+      break;
     default:
       if (token.IsBinaryOperator())
-        char_format = GetOperatorCharFormat();
+        char_format = GetSpecialCharFormat();
       break;
     }
 
     if (char_format.has_value())
-      setFormat(int(token.string_position), int(token.string_length), *char_format);
+      set_block_format(int(token.string_position), int(token.string_length), *char_format);
   }
 
+  // This doesn't need to be run for every "block", but it works.
   if (ciface::ExpressionParser::ParseStatus::Successful != tokenize_status)
   {
     m_result_text->setText(tr("Invalid Token."));
   }
   else
   {
+    ciface::ExpressionParser::RemoveInertTokens(&tokens);
     const auto parse_status = ciface::ExpressionParser::ParseTokens(tokens);
 
     m_result_text->setText(
@@ -155,7 +178,8 @@ void ControlExpressionSyntaxHighlighter::highlightBlock(const QString& text)
     if (ciface::ExpressionParser::ParseStatus::Successful != parse_status.status)
     {
       const auto token = *parse_status.token;
-      setFormat(int(token.string_position), int(token.string_length), GetInvalidCharFormat());
+      set_block_format(int(token.string_position), int(token.string_length),
+                       GetInvalidCharFormat());
     }
   }
 }

Source/Core/InputCommon/ControlReference/ExpressionParser.cpp

@@ -87,6 +87,14 @@ Token Lexer::GetRealLiteral(char first_char)
   return Token(TOK_INVALID);
 }
 
+Token Lexer::PeekToken()
+{
+  const auto old_it = it;
+  const auto tok = NextToken();
+  it = old_it;
+  return tok;
+}
+
 Token Lexer::NextToken()
 {
   if (it == expr.end())
@@ -99,7 +107,7 @@ Token Lexer::NextToken()
   case '\t':
   case '\n':
   case '\r':
-    return Token(TOK_DISCARD);
+    return Token(TOK_WHITESPACE);
   case '(':
     return Token(TOK_LPAREN);
   case ')':
@@ -154,8 +162,19 @@ ParseStatus Lexer::Tokenize(std::vector<Token>& tokens)
     tok.string_position = string_position;
     tok.string_length = it - expr.begin();
 
-    if (tok.type == TOK_DISCARD)
-      continue;
+    // Handle /* */ style comments.
+    if (tok.type == TOK_DIV && PeekToken().type == TOK_MUL)
+    {
+      const auto end_of_comment = expr.find("*/", it - expr.begin());
+
+      if (end_of_comment == std::string::npos)
+        return ParseStatus::SyntaxError;
+
+      tok.type = TOK_COMMENT;
+      tok.string_length = end_of_comment + 4;
+
+      it = expr.begin() + end_of_comment + 2;
+    }
 
     tokens.push_back(tok);
 
@@ -671,9 +690,19 @@ static ParseResult ParseComplexExpression(const std::string& str)
   if (tokenize_status != ParseStatus::Successful)
     return ParseResult::MakeErrorResult(Token(TOK_INVALID), _trans("Tokenizing failed."));
 
+  RemoveInertTokens(&tokens);
   return ParseTokens(tokens);
 }
 
+void RemoveInertTokens(std::vector<Token>* tokens)
+{
+  tokens->erase(std::remove_if(tokens->begin(), tokens->end(),
+                               [](const Token& tok) {
+                                 return tok.type == TOK_COMMENT || tok.type == TOK_WHITESPACE;
+                               }),
+                tokens->end());
+}
+
 static std::unique_ptr<Expression> ParseBarewordExpression(const std::string& str)
 {
   ControlQualifier qualifier;

Source/Core/InputCommon/ControlReference/ExpressionParser.h

@@ -15,7 +15,7 @@ namespace ciface::ExpressionParser
 {
 enum TokenType
 {
-  TOK_DISCARD,
+  TOK_WHITESPACE,
   TOK_INVALID,
   TOK_EOF,
   TOK_LPAREN,
@@ -25,6 +25,7 @@ enum TokenType
   TOK_LITERAL,
   TOK_VARIABLE,
   TOK_BAREWORD,
+  TOK_COMMENT,
   // Binary Ops:
   TOK_BINARY_OPS_BEGIN,
   TOK_AND = TOK_BINARY_OPS_BEGIN,
@@ -95,6 +96,7 @@ private:
   Token GetBareword(char c);
   Token GetRealLiteral(char c);
 
+  Token PeekToken();
   Token NextToken();
 };
 
@@ -186,5 +188,6 @@ private:
 
 ParseResult ParseExpression(const std::string& expr);
 ParseResult ParseTokens(const std::vector<Token>& tokens);
+void RemoveInertTokens(std::vector<Token>* tokens);
 
 }  // namespace ciface::ExpressionParser