Add `--preproc-token-limit` (#5768)

bin/verilator

@@ -426,6 +426,7 @@ detailed descriptions of these arguments.
     --pipe-filter <command>     Filter all input through a script
     --pp-comments               Show preprocessor comments with -E
     --prefix <topname>          Name of top-level class
+    --preproc-token-limit       Maximum tokens on a line allowed by preprocessor
     --private                   Debugging; see docs
     --prof-c                    Compile C++ code with profiling
     --prof-cfuncs               Name functions for profiling

docs/guide/exe_verilator.rst

@@ -1142,6 +1142,12 @@ Summary:
    prepended to the name of the :vlopt:`--top` option, or V prepended to
    the first Verilog filename passed on the command line.
 
+.. option:: --preproc-token-limit
+
+   Rarely needed. Configure the limit of the number of tokens Verilator
+   can process on a single line to prevent infinite loops and other hangs.
+   Defaults to 40000 tokens.
+
 .. option:: --private
 
    Opposite of :vlopt:`--public`.  This is the default; this option exists for

src/V3Options.cpp

@@ -1502,6 +1502,10 @@ void V3Options::parseOptsList(FileLine* fl, const string& optdir, int argc,
         validateIdentifier(fl, valp, "--prefix");
         m_prefix = valp;
     });
+    DECL_OPTION("-preproc-token-limit", CbVal, [this, fl](const char* valp) {
+        m_preprocTokenLimit = std::atoi(valp);
+        if (m_preprocTokenLimit <= 0) fl->v3error("--preproc-token-limit must be > 0: " << valp);
+    });
     DECL_OPTION("-private", CbCall, [this]() { m_public = false; });
     DECL_OPTION("-prof-c", OnOff, &m_profC);
     DECL_OPTION("-prof-cfuncs", CbCall, [this]() { m_profC = m_profCFuncs = true; });

src/V3Options.h

@@ -324,6 +324,7 @@ private:
     int         m_outputSplitCFuncs = -1;  // main switch: --output-split-cfuncs
     int         m_outputSplitCTrace = -1;  // main switch: --output-split-ctrace
     int         m_pinsBv = 65;       // main switch: --pins-bv
+    int         m_preprocTokenLimit = 40000; // main switch: --preproc-token-limit
     int         m_publicDepth = 0;   // main switch: --public-depth
     int         m_reloopLimit = 40; // main switch: --reloop-limit
     VOptionBool m_skipIdentical;  // main switch: --skip-identical
@@ -465,6 +466,7 @@ public:
     bool preprocOnly() const { return m_preprocOnly; }
     bool makePhony() const { return m_makePhony; }
     bool preprocNoLine() const { return m_preprocNoLine; }
+    int preprocTokenLimit() const { return m_preprocTokenLimit; }
     bool underlineZero() const { return m_underlineZero; }
     string flags() const { return m_flags; }
     bool systemC() const VL_MT_SAFE { return m_systemC; }

src/V3PreProc.cpp

@@ -963,9 +963,9 @@ int V3PreProcImp::getRawToken() {
         if (m_lastLineno != m_lexp->m_tokFilelinep->lineno()) {
             m_lastLineno = m_lexp->m_tokFilelinep->lineno();
             m_tokensOnLine = 0;
-        } else if (++m_tokensOnLine > LINE_TOKEN_MAX) {
-            error("Too many preprocessor tokens on a line (>" + cvtToStr(LINE_TOKEN_MAX)
-                  + "); perhaps recursive `define");
+        } else if (++m_tokensOnLine > v3Global.opt.preprocTokenLimit()) {
+            error("Too many preprocessor tokens on a line (>"
+                  + cvtToStr(v3Global.opt.preprocTokenLimit()) + "); perhaps recursive `define");
             tok = VP_EOF_ERROR;
         }
 

src/V3PreProc.h

@@ -42,7 +42,6 @@ public:
     // CONSTANTS
     enum MiscConsts {
         DEFINE_RECURSION_LEVEL_MAX = 1000,  // How many `def substitutions before an error
-        LINE_TOKEN_MAX = 40000,  // How many tokens on a line before an error
         INCLUDE_DEPTH_MAX = 500,  // How many `includes deep before an error
         // Streams deep (sometimes `def deep) before an error.
         // Set more than DEFINE_RECURSION_LEVEL_MAX or INCLUDE_DEPTH_MAX.

test_regress/t/t_flag_values_bad.out

@@ -1,4 +1,5 @@
 %Error: --output-split-cfuncs must be >= 0: -1
 %Error: --output-split-ctrace must be >= 0: -1
+%Error: --preproc-token-limit must be > 0: 0
 %Error: --reloop-limit must be >= 2: -1
 %Error: Exiting due to

test_regress/t/t_flag_values_bad.py

@@ -11,9 +11,11 @@ import vltest_bootstrap
 
 test.scenarios('vlt')
 
-test.lint(
-    verilator_flags2=["--output-split-cfuncs -1", "--output-split-ctrace -1", "--reloop-limit -1"],
-    fails=True,
-    expect_filename=test.golden_filename)
+test.lint(verilator_flags2=[
+    "--output-split-cfuncs -1", "--output-split-ctrace -1", "--preproc-token-limit 0",
+    "--reloop-limit -1"
+],
+          fails=True,
+          expect_filename=test.golden_filename)
 
 test.passes()

test_regress/t/t_pp_circ_subst_bad2.out

@@ -0,0 +1,3 @@
+%Error: t/t_pp_circ_subst_bad.v:8:40002: Too many preprocessor tokens on a line (>20000); perhaps recursive `define
+%Error: t/t_pp_circ_subst_bad.v:8:1: syntax error, unexpected IDENTIFIER
+%Error: Exiting due to

test_regress/t/t_pp_circ_subst_bad2.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# DESCRIPTION: Verilator: Verilog Test driver/expect definition
+#
+# Copyright 2025 by Wilson Snyder. This program is free software; you
+# can redistribute it and/or modify it under the terms of either the GNU
+# Lesser General Public License Version 3 or the Perl Artistic License
+# Version 2.0.
+# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
+
+import vltest_bootstrap
+
+test.scenarios('linter')
+test.top_filename = "t/t_pp_circ_subst_bad.v"
+
+test.lint(verilator_flags2=["--preproc-token-limit 20000"],
+          fails=True,
+          expect_filename=test.golden_filename)
+
+test.passes()