Merge pull request #4 from xatavian/topic-improve-dbc-parsing

Better parsing robustness for DBC file format

Author: xatavian

CMakeLists.txt

@@ -12,8 +12,9 @@ set(CPPPARSER_INCLUDE_DIRECTORY
 set(CPPPARSER_SRC_FILES
 	src/models/CANDatabase.cpp
 	src/models/CANFrame.cpp
-  	src/models/CANSignal.cpp
+  src/models/CANSignal.cpp
 	src/parsing/DBCParser.cpp
+	src/parsing/ParsingUtils.cpp
 	src/parsing/Tokenizer.cpp
 	src/analysis/CANFrameAnalysis.cpp)
 

README.md

@@ -112,7 +112,7 @@ C++ CAN Parser currently is compiled as a static library.
 
 Different uses of `can-parse` are possible, mainly 4 operations are included in `can-parse`:
 * Print a summary of the whole database
-* Print a detailed view of a single entry of the database (very basic implementation for now)
+* Print a detailed view of a single entry of the database
   * CAN ID, DLC, Period, Comment
   * Signals' description
     * Name

include/ParsingUtils.h

@@ -6,80 +6,33 @@
 #include <iostream>
 #include "CANDatabaseException.h"
 
-void throwError(const std::string& category, const std::string& description,
-                unsigned long long line) {
-  throw CANDatabaseException(
-    category + ": " + description + " at line " + std::to_string(line)
-  );
-}
+void throw_error(const std::string& category, const std::string& description,
+                unsigned long long line);
 
-void warning(const std::string& description, unsigned long long line) {
-  std::cout << "WARNING: "
-            << description
-            << " at line "
-            << line
-            << std::endl;
-}
+void warning(const std::string& description, unsigned long long line) ;
 
-void skipIf(Tokenizer& tokenizer, const std::string& token) {
-  if(tokenizer.getNextToken().image() == token)
-    return;
+const Token&
+assert_token(Tokenizer& tokenizer, const std::string& token);
 
-  throwError(
-    "Syntax error",
-    "expected \"" + token + "\" but got \"" +
-      tokenizer.getCurrentToken().image() + "\"",
-    tokenizer.lineCount()
-  );
-}
+const Token&
+assert_token(Tokenizer& tokenizer, Token::Type targetType);
 
-void assertToken(const Tokenizer& tokenizer, const std::string& token) {
-  if(tokenizer.getCurrentToken().image() == token)
-    return;
+const Token&
+assert_current_token(const Tokenizer& tokenizer, const std::string& token);
 
-  throwError(
-    "Syntax error",
-    "expected \"" + token + "\" but got \"" +
-      tokenizer.getCurrentToken().image() + "\"",
-    tokenizer.lineCount()
-  );
-}
+const Token&
+assert_current_token(const Tokenizer& tokenizer, Token::Type type);
 
+bool is_current_token(const Tokenizer& tokenizer, const std::string& token);
 
-Token checkCurrentTokenType(const Token& toCheck, Token::Type targetType,
-                            unsigned long long line) {
-  if(toCheck.type() == targetType)
-    return toCheck;
+bool is_current_token(const Tokenizer& tokenizer, Token::Type token);
 
-  throwError(
-    "Syntax error",
-    "unexpected \"" + toCheck.image() + "\"",
-    line
-  );
+bool is_token(Tokenizer& tokenizer, const std::string& token);
 
-  // Removes a compilation warning
-  return toCheck;
-}
+bool is_token(Tokenizer& tokenizer, Token::Type token);
 
-Token checkCurrentTokenType(const Token& toCheck,
-                            const std::string& token,
-                            unsigned long long line) {
-  if(toCheck.image() == token)
-    return toCheck;
+bool peek_token(Tokenizer& tokenizer, Token::Type type);
 
-  throwError(
-    "Syntax error",
-    "unexpected \"" + toCheck.image() + "\"",
-    line
-  );
+bool peek_token(Tokenizer& tokenizer, const std::string& token);
 
-  // Removes a compilation warning
-  return toCheck;
-}
-
-Token checkTokenType(Tokenizer& tokenizer, Token::Type targetType) {
-  return checkCurrentTokenType(tokenizer.getNextToken(),
-                               targetType,
-                               tokenizer.lineCount());
-}
 #endif

include/Token.h

@@ -3,46 +3,120 @@
 
 #include <string>
 
-class Token {
-public:
+struct Token {
+  /**
+   * @brief The different kinds of tokens
+   * The positive and negative numbers represent different types
+   * of tokens but a grammar rule that accepts any kind of number
+   * can use the Number type
+   */
   enum Type {
-      Literal,
+      StringLiteral,
       Separator,
+      PositiveNumber,
+      NegativeNumber,
+      FloatingPointNumber,
       Number,
       Identifier,
-      Sign,
+      ArithmeticSign,
       Eof
   };
 
-  Token() = default;
-  Token(Type type, const std::string& image) :
-   type_(type), image_(image) {}
+  /**
+   * @return A Token representing an arithmetic sign
+   */
+  static Token createArithmeticSign(char src);
+  
+  /**
+   * @return A Token representing a separator
+   */
+  static Token createSeparator(char src);
+
+  /**
+   * The Token's type returned here depends on is_positive and is_float
+   * @param number The image of the number
+   * @param is_positive true if number represents a positive number
+   * @param is_float true if the number represents a floating-point number
+   * @return A Token representing a number
+   */
+  static Token createNumber(const std::string& number, bool is_positive, bool is_float);
+  
+  /**
+   * @brief Constructs an EOF token.
+   */
+  Token();
+
+  /**
+   * @brief Construct a new token with the given type and image
+   */
+  Token(Type type, const std::string& image = "");
+  
   Token(const Token&) = default;
   Token& operator=(const Token&) = default;
+  Token(Token&&) = default;
+  Token& operator=(Token&&) = default;
 
-  Type type() const {
-    return type_;
-  }
-
-  const std::string& image() const {
-    return image_;
-  }
+  /**
+   * @brief Equality comparison with another token
+   * @see operator== with string and Token::Type
+   */
+  bool operator==(const Token&) const;
+  
+  /**
+   * @return true if the token's image is equal to other
+   * @param other String to compare
+   */
+  bool operator==(const std::string&) const;
+  
+  /**
+   * If the token's type is Number or if other is Number,
+   * then it also matches PositiveNumber, NegativeNumber and FloatingPointNumber
+   *  
+   * @return true if the token's type is equal to other
+   * @param other Type to compare
+   */
+  bool operator==(Token::Type) const;
 
-  unsigned long long toUInt() const {
-    return std::stoul(image_);
-  }
+  /**
+   * @see operator==
+   */
+  bool operator!=(const Token&) const;
 
-  long long toInt() const {
-    return std::stol(image_);
-  }
+  /**
+   * @see operator==
+   */
+  bool operator!=(const std::string&) const;
+  
+  /**
+   * @see operator==
+   */
+  bool operator!=(Token::Type) const;
 
-  double toDouble() const {
-    return std::stod(image_);
-  }
+  /**
+   * Using the standard library's functionnalities, parses the 
+   * image to an unsigned integer
+   */
+  unsigned long long toUInt() const;
+  
+  /**
+   * Same as above but outputs a signed integer
+   */
+  long long toInt() const;
+  
+  /**
+   * Same as above but outputs a floating-point number
+   */
+  double toDouble() const;
+  
+  /**
+   * @brief The token's type
+   */
+  Type type;
 
-private:
-  Type type_;
-  std::string image_;
+  /**
+   * @brief The token's image, ie. the string that represents it
+   */
+  std::string image;
 };
 
 #endif

include/Tokenizer.h

@@ -11,12 +11,13 @@ class Tokenizer {
   Tokenizer();
   virtual ~Tokenizer() = default;
 
-  Token getNextToken();
-  Token getCurrentToken() const;
+  const Token& getNextToken();
+  const Token& getCurrentToken() const;
 
   void skipLine();
   void skipUntil(const std::string& token);
   void saveToken(const Token& token);
+  void saveTokenIfNotEof(const Token& token);
 
   unsigned long long charCount() const;
   unsigned long long lineCount() const;
@@ -25,6 +26,8 @@ class Tokenizer {
   char getNextChar();
   char getCurrentChar() const;
 
+  std::string parseNumber(bool& is_float);
+
 private:
   virtual char doGetNextChar() = 0;