[PATCH 15/15] Support for 64-bit location_t: Configury parts

[Lewis Hyatt]

Add --enable-large-source-locations (off by default for now) to enable
64-bit location_t.

gcc/ChangeLog:

* configure.ac: Add --enable-large-source-locations.
* config.in: Regenerate.
* configure: Regenerate.
* doc/install.texi: Document the new option.

libcpp/ChangeLog:

* configure.ac: Add --enable-large-source-locations.
* config.in: Regenerate.
* configure: Regenerate.
---
 libcpp/config.in |  3 +++
 libcpp/configure | 17 +
 libcpp/configure.ac  | 11 +++
 gcc/config.in|  6 ++
 gcc/configure| 21 +++--
 gcc/configure.ac | 11 +++
 gcc/doc/install.texi | 15 +++
 7 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/libcpp/config.in b/libcpp/config.in
index b2e2f4e842c..74ff169a322 100644
--- a/libcpp/config.in
+++ b/libcpp/config.in
@@ -20,6 +20,9 @@
 /* Define to enable system headers canonicalization. */
 #undef ENABLE_CANONICAL_SYSTEM_HEADERS
 
+/* Define to enable 64-bit locations. */
+#undef ENABLE_LARGE_SOURCE_LOCATIONS
+
 /* Define to 1 if translation of program messages to the user's native
language is requested. */
 #undef ENABLE_NLS
diff --git a/libcpp/configure b/libcpp/configure
index 1391081ba09..eca82ab568f 100755
--- a/libcpp/configure
+++ b/libcpp/configure
@@ -753,6 +753,7 @@ enable_host_shared
 enable_host_pie
 enable_cet
 enable_valgrind_annotations
+enable_large_source_locations
 '
   ac_precious_vars='build_alias
 host_alias
@@ -1397,6 +1398,8 @@ Optional Features:
   --enable-cetenable Intel CET in host libraries [default=auto]
   --enable-valgrind-annotations
   enable valgrind runtime interaction
+  --enable-large-source-locations
+  enable 64-bit source locations
 
 Optional Packages:
   --with-PACKAGE[=ARG]use PACKAGE [ARG=yes]
@@ -9416,6 +9419,20 @@ $as_echo "#define ENABLE_VALGRIND_WORKAROUNDS 1" 
>>confdefs.h
 
 fi
 
+# Specify whether to use 64-bit locations
+# Check whether --enable-large-source-locations was given.
+if test "${enable_large_source_locations+set}" = set; then :
+  enableval=$enable_large_source_locations;
+else
+  enable_large_source_locations=no
+fi
+
+if test x$enable_large_source_locations != xno; then
+
+$as_echo "#define ENABLE_LARGE_SOURCE_LOCATIONS 1" >>confdefs.h
+
+fi
+
 # Output.
 
 ac_config_headers="$ac_config_headers config.h:config.in"
diff --git a/libcpp/configure.ac b/libcpp/configure.ac
index 981f97c4abd..091bbc44537 100644
--- a/libcpp/configure.ac
+++ b/libcpp/configure.ac
@@ -246,6 +246,17 @@ if test x$enable_valgrind_annotations != xno \
  possible memory leaks because of libcpp use of interior pointers.])
 fi
 
+# Specify whether to use 64-bit locations
+AC_ARG_ENABLE([large-source-locations],
+[AC_HELP_STRING([--enable-large-source-locations],
+[enable 64-bit source locations])],
+[],
+enable_large_source_locations=no)
+if test x$enable_large_source_locations != xno; then
+  AC_DEFINE(ENABLE_LARGE_SOURCE_LOCATIONS,
+1, [Define to enable 64-bit locations.])
+fi
+
 # Output.
 
 AC_CONFIG_HEADERS(config.h:config.in, [echo timestamp > stamp-h1])
diff --git a/gcc/config.in b/gcc/config.in
index 0a506d1783a..ed779e78a12 100644
--- a/gcc/config.in
+++ b/gcc/config.in
@@ -206,6 +206,12 @@
 #endif
 
 
+/* Define to enable 64-bit locations. */
+#ifndef USED_FOR_TARGET
+#undef ENABLE_LARGE_SOURCE_LOCATIONS
+#endif
+
+
 /* Define if gcc should always pass --build-id to linker. */
 #ifndef USED_FOR_TARGET
 #undef ENABLE_LD_BUILDID
diff --git a/gcc/configure b/gcc/configure
index 150ab616414..490f070c820 100755
--- a/gcc/configure
+++ b/gcc/configure
@@ -1059,6 +1059,7 @@ with_diagnostics_urls
 enable_default_pie
 enable_cet
 enable_s390_excess_float_precision
+enable_large_source_locations
 '
   ac_precious_vars='build_alias
 host_alias
@@ -1832,6 +1833,8 @@ Optional Features:
   --enable-s390-excess-float-precision
   on s390 targets, evaluate float with double
   precision when in standards-conforming mode
+  --enable-large-source-locations
+  enable 64-bit source locations
 
 Optional Packages:
   --with-PACKAGE[=ARG]use PACKAGE [ARG=yes]
@@ -21454,7 +21457,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21457 "configure"
+#line 21460 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -21560,7 +21563,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 21563 "configure"
+#line 21566 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -34912,6 +34915,20 @@ _ACEOF
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $fhardened_support" >&5
 $as_echo "$fhardened_support" >&6; }
 
+# Specify whether to use 64-bit locations
+# Check whether 

[PATCH 13/15] Support for 64-bit location_t: Internal parts

[Lewis Hyatt]

Several of the selftests in diagnostic-show-locus.cc and input.cc are
sensitive to linemap internals. Adjust them here so they will support 64-bit
location_t if configured.

Likewise, handle 64-bit location_t in the support for
-fdump-internal-locations. As was done with the analyzer, convert to
(unsigned long long) explicitly so that 32- and 64-bit can be handled with
the same printf formats.

gcc/ChangeLog:

* diagnostic-show-locus.cc
(test_one_liner_fixit_validation_adhoc_locations): Adapt so it can
effectively test 7-bit ranges instead of 5-bit ranges.
(test_one_liner_fixit_validation_adhoc_locations_utf8): Likewise.
* input.cc (get_end_location): Adjust types to support 64-bit
location_t.
(write_digit_row): Likewise.
(dump_location_range): Likewise.
(dump_location_info): Likewise.
(class line_table_case): Likewise.
(test_accessing_ordinary_linemaps): Replace some hard-coded
constants with the values defined in line-map.h.
(for_each_line_table_case): Likewise.
---
 gcc/diagnostic-show-locus.cc | 128 +--
 gcc/input.cc | 100 ++-
 2 files changed, 157 insertions(+), 71 deletions(-)

diff --git a/gcc/diagnostic-show-locus.cc b/gcc/diagnostic-show-locus.cc
index a2a4b047ff9..ffe9e807104 100644
--- a/gcc/diagnostic-show-locus.cc
+++ b/gcc/diagnostic-show-locus.cc
@@ -4013,12 +4013,12 @@ test_one_liner_fixit_validation_adhoc_locations ()
 {
   /* Generate a range that's too long to be packed, so must
  be stored as an ad-hoc location (given the defaults
- of 5 bits or 0 bits of packed range); 41 columns > 2**5.  */
+ of 5 or 7 bits or 0 bits of packed range); 150 columns > 2**7.  */
   const location_t c7 = linemap_position_for_column (line_table, 7);
-  const location_t c47 = linemap_position_for_column (line_table, 47);
-  const location_t loc = make_location (c7, c7, c47);
+  const location_t c157 = linemap_position_for_column (line_table, 157);
+  const location_t loc = make_location (c7, c7, c157);
 
-  if (c47 > LINE_MAP_MAX_LOCATION_WITH_COLS)
+  if (c157 > LINE_MAP_MAX_LOCATION_WITH_COLS)
 return;
 
   ASSERT_TRUE (IS_ADHOC_LOC (loc));
@@ -4032,7 +4032,18 @@ test_one_liner_fixit_validation_adhoc_locations ()
 
 test_diagnostic_context dc;
 ASSERT_STREQ (" foo = bar.field;\n"
- "   ^~   \n"
+ "   ^~   "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  \n"
  "   test\n",
  dc.test_show_locus (richloc));
   }
@@ -4040,29 +4051,62 @@ test_one_liner_fixit_validation_adhoc_locations ()
   /* Remove.  */
   {
 rich_location richloc (line_table, loc);
-source_range range = source_range::from_locations (loc, c47);
+source_range range = source_range::from_locations (loc, c157);
 richloc.add_fixit_remove (range);
 /* It should not have been discarded by the validator.  */
 ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
 test_diagnostic_context dc;
 ASSERT_STREQ (" foo = bar.field;\n"
- "   ^~   \n"
- "   -\n",
+ "   ^~   "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  "
+ "  \n"
+ "   -"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--"
+ "--\n",
  dc.test_show_locus (richloc));
   }
 
   /* Replace.  */
   {
 rich_location richloc (line_table, loc);
-source_range range = source_range::from_locations (loc, c47);
+source_range range = source_range::from_locations (loc, c157);
 richloc.add_fixit_replace (range, "test");
 /* It should not have been discarded by the validator.  */
 ASSERT_EQ (1, richloc.get_num_fixit_hints ());
 
 test_diagnostic_context dc;
 ASSERT_STREQ (" foo = bar.field;\n"
- "   ^~~

[PATCH 11/15] Support for 64-bit location_t: RTL parts

[Lewis Hyatt]

Some RTL objects need to store a location_t. Currently, they store it in the
rt_int field of union rtunion, but in a world where location_t could be
64-bit, they need to store it in a larger variable. Unfortunately, rtunion
does not currently have a 64-bit int type for that purpose, so add one. In
order to avoid increasing any overhead when 64-bit locations are not in use,
the new field is dedicated for location_t storage only and has type
"location_t" so it will only be 64-bit if necessary. This necessitates
adding a new RTX format code 'L' for locations. There are very many switch
statements in the codebase that inspect the RTX format code. I took the
approach of finding all of them that handle code 'i' or 'n' and making sure
they handle 'L' too. I am sure that some of these call sites can never see
an 'L' code, but I thought it would be safer and more future-proof to handle
as many as possible, given it's just a line or two to add in most cases.

While testing this with --enable-checking=rtl, I came across one place in
final.cc that seems to be a (currently) harmless misuse of RTL:

set_cur_block_to_this_block:
  if (! this_block)
{
  if (INSN_LOCATION (insn) == UNKNOWN_LOCATION)
continue;
  else
this_block = DECL_INITIAL (cfun->decl);
}

In this part of reemit_insn_block_notes(), the insn variable could actually
be a NOTE and not an INSN. In that case, INSN_LOCATION() shouldn't be
called on it. It works fine currently because the field is properly accessed
by XINT() either way. (For an INSN, it is a location, but for a NOTE, it is
the note type enum). Currently, if insn is a NOTE, the comparison must
always be false because the note type is not equal to
0==UNKNOWN_LOCATION. Once locations and ints are differentiated, this line
leads to a checking failure, which I resolved by checking for the NOTE_P
case before calling INSN_LOCATION.

gcc/ChangeLog:

* rtl.def (DEBUG_INSN): Use new format code 'L' for location_t fields.
(INSN): Likewise.
(JUMP_INSN): Likewise.
(CALL_INSN): Likewise.
(ASM_INPUT): Likewise.
(ASM_OPERANDS): Likewise.
* rtl.h (union rtunion): Add new location_t RT_LOC member for use by
the 'L' format.
(struct rtx_debug_insn): Adjust comment.
(struct rtx_nonjump_insn): Adjust comment.
(struct rtx_call_insn): Adjust comment.
(XLOC): New accessor macro for rtunion::rt_loc.
(X0LOC): Likewise.
(XCLOC): Likewise.
(INSN_LOCATION): Use XLOC instead of XUINT to retrieve a location_t.
(NOTE_MARKER_LOCATION): Likewise for XCUINT -> XCLOC.
(ASM_OPERANDS_SOURCE_LOCATION): Likewise.
(ASM_INPUT_SOURCE_LOCATION):Likewise.
(gen_rtx_ASM_INPUT): Adjust to use sL format instead of si.
(gen_rtx_INSN): Adjust prototype to use location_r rather than int
for the location.
* cfgrtl.cc (force_nonfallthru_and_redirect): Change type of LOC
local variable from int to location_t.
* rtlhash.cc (add_rtx): Support 'L' format in the switch statement.
* var-tracking.cc (loc_cmp): Likewise.
* alias.cc (rtx_equal_for_memref_p): Likewise.
* config/alpha/alpha.cc (summarize_insn): Likewise.
* config/ia64/ia64.cc (rtx_needs_barrier): Likewise.
* config/rs6000/rs6000.cc (rs6000_hash_constant): Likewise.
* cse.cc (hash_rtx): Likewise.
(exp_equiv_p): Likewise.
* cselib.cc (rtx_equal_for_cselib_1): Likewise.
(cselib_hash_rtx): Likewise.
(cselib_expand_value_rtx_1): Likewise.
* emit-rtl.cc (copy_insn_1): Likewise.
(gen_rtx_INSN): Change the location argument from int to location_t,
and call the corresponding gen_rtf_fmt_* function.
* final.cc (reemit_insn_block_notes): Don't call INSN_LOCATION if
NOTE_P; the field being accessed is not a location in this case.
(leaf_renumber_regs_insn): Support 'L' format in the switch statement.
* genattrtab.cc (attr_rtx_1): Likewise.
* genemit.cc (gen_exp): Likewise.
* gengenrtl.cc (type_from_format): Likewise.
(accessor_from_format): Likewise.
* gengtype.cc (adjust_field_rtx_def): Likewise.
* genpeep.cc (match_rtx): Likewise; just mark gcc_unreachable() for
now.
* genrecog.cc (find_operand): Support 'L' format in the switch 
statement.
(find_matching_operand): Likewise.
(validate_pattern): Likewise.
* gensupport.cc (subst_pattern_match): Likewise.
(get_alternatives_number): Likewise.
(collect_insn_data): Likewise.
(alter_predicate_for_insn): Likewise.
(alter_constraints): Likewise.
(subst_dup): Likewise.
* jump.cc (rtx_renumbered_equal_p): Likewise.
* loop-invariant.cc (hash_invariant_expr_1): Likewise.
* lra-constraints.cc (operands_match_p): Likewise.
* lra.cc (l

[PATCH 07/15] Support for 64-bit location_t: toplev parts

[Lewis Hyatt]

The recommended bits reserved in a line_map to store ranges has always been
5, meaning that identifiers up to length 32 can be stored without generating
an ad-hoc location. When 64-bit location_t is configured, there are plenty
of bits to go around, and so the recommended default is larger. line-map.h
now exports the recommended setting based on what was configured, so make
use of that when setting up the line map rather than hard-coding 5.

Also silently ignore -flarge-source-files when configured with 64-bit
location_t. That flag is a workaround for cases where we run out of 32-bit
location_t entries; it effectively makes every token's location an ad-hoc
location, making more of the location_t space available for tracking source
locations. This workaround is not necessary with a 64-bit location_t.

gcc/ChangeLog:

* toplev.cc (general_init): Use constant
line_map_suggested_range_bits instead of hard-coded integer.
(process_options): Silently ignore -flarge-source-files if 64-bit
location_t is in use.
---
 gcc/toplev.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gcc/toplev.cc b/gcc/toplev.cc
index 779049674b4..655be409a36 100644
--- a/gcc/toplev.cc
+++ b/gcc/toplev.cc
@@ -1137,7 +1137,7 @@ general_init (const char *argv0, bool init_signals, 
unique_argv original_argv)
   linemap_init (line_table, BUILTINS_LOCATION);
   line_table->m_reallocator = realloc_for_line_map;
   line_table->m_round_alloc_size = ggc_round_alloc_size;
-  line_table->default_range_bits = 5;
+  line_table->default_range_bits = line_map_suggested_range_bits;
   init_ttree ();
 
   /* Initialize register usage now so switches may override.  */
@@ -1765,8 +1765,10 @@ process_options ()
 hash_table_sanitize_eq_limit
   = param_hash_table_verification_limit;
 
+  #ifndef ENABLE_LARGE_SOURCE_LOCATIONS
   if (flag_large_source_files)
 line_table->default_range_bits = 0;
+  #endif
 
   diagnose_options (&global_options, &global_options_set, UNKNOWN_LOCATION);
 

[PATCH 10/15] Support for 64-bit location_t: C++ modules parts

[Lewis Hyatt]

The modules implementation is necessarily sensitive to the internal workings
of class line_map, and so it needed changes in order to handle a 64-bit
location_t. The changes mostly boil down to supporting that in the debug
dumping routines (which is accomplished by using a new custom code %K for
that purpose), and supporting that when streaming in and out from the
module (which is accomplished by using a new loc() function to go along with
existing abstractions like u() or z() for streaming in and out different
data types).

gcc/cp/ChangeLog:

* module.cc (bytes_out::loc): New function.
(bytes_in::loc): New function.
(struct span): Change int fields to location_diff_t.
(range_t): Change from "unsigned int" to "line_map_uint_t".
(struct ord_loc_info): Likewise.
(struct macro_loc_info): Likewise.
(class module_state): Likewise.
(dumper::operator()): Add new code 'K' for dumping a location_t.
(loc_spans::init): Use %K instead of %u for location_t dumps.
(loc_spans::open): Likewise.
(loc_spans::close): Likewise. Adjust bitwise expressions to support
64-bit location_t as well.
(struct module_state_config): Change ordinary_locs and macro_locs
from "unsigned int" to "line_map_uint_t".  Reorder fields to improve
packing.  Rather than changing the constructor initializer list to
match the new order, switch to NSDMI instead.
(module_state::note_location): Adjust to support 64-bit location_t.
(module_state::write_location): Use %K instead of %u for location_t
dumps. Use loc() instead of u() for streaming location_t.
(module_state::read_location): Likewise.
(module_state::write_ordinary_maps): Likewise.
(module_state::write_macro_maps): Likewise.
(module_state::write_config): Likewise.
(module_state::read_config): Likewise.
(module_state::write_prepare_maps): Use %K instead of %u for
location_t dumps. Adjust variable types and bitwise expressions to
support 64-bit location_t.
(module_state::read_ordinary_maps): Likewise.
(module_state::read_macro_maps): Likewise.
(preprocess_module): Adjust data types to support 64-bit number of
line maps.
---
 gcc/cp/module.cc | 229 +++
 1 file changed, 130 insertions(+), 99 deletions(-)

diff --git a/gcc/cp/module.cc b/gcc/cp/module.cc
index dde7e5f6dbf..415bdd41ea3 100644
--- a/gcc/cp/module.cc
+++ b/gcc/cp/module.cc
@@ -350,6 +350,9 @@ typedef hash_map 
ptr_int_hash_map;
 /* Variable length buffer.  */
 
 namespace {
+
+constexpr line_map_uint_t loc_one = 1;
+
 class data {
 public:
   class allocator {
@@ -549,6 +552,7 @@ public:
   int i ();/* Read a signed int.  */
   unsigned u ();   /* Read an unsigned int.  */
   size_t z (); /* Read a size_t.  */
+  location_t loc ();/* Read a location_t.  */
   HOST_WIDE_INT wi ();  /* Read a HOST_WIDE_INT.  */
   unsigned HOST_WIDE_INT wu (); /* Read an unsigned HOST_WIDE_INT.  */
   const char *str (size_t * = NULL); /* Read a string.  */
@@ -633,6 +637,7 @@ public:
   void i (int);/* Write signed int.  */
   void u (unsigned);   /* Write unsigned int.  */
   void z (size_t s);   /* Write size_t.  */
+  void loc (location_t); /* Write location_t.  */
   void wi (HOST_WIDE_INT); /* Write HOST_WIDE_INT.  */
   void wu (unsigned HOST_WIDE_INT);  /* Write unsigned HOST_WIDE_INT.  */
   void str (const char *ptr)
@@ -1057,6 +1062,26 @@ bytes_in::z ()
 return wu ();
 }
 
+/* location_t written as 32- or 64-bit as needed.  */
+
+inline void bytes_out::loc (location_t l)
+{
+#ifdef ENABLE_LARGE_SOURCE_LOCATIONS
+  wu (l);
+#else
+  u (l);
+#endif
+}
+
+inline location_t bytes_in::loc ()
+{
+#ifdef ENABLE_LARGE_SOURCE_LOCATIONS
+  return wu ();
+#else
+  return u ();
+#endif
+}
+
 /* Buffer simply memcpied.  */
 void *
 bytes_out::buf (size_t len)
@@ -3210,7 +3235,7 @@ trees_out::~trees_out ()
 
 
 /* I use half-open [first,second) ranges.  */
-typedef std::pair range_t;
+typedef std::pair range_t;
 
 /* A range of locations.  */
 typedef std::pair loc_range_t;
@@ -3227,8 +3252,9 @@ public:
   struct span {
 loc_range_t ordinary;  /* Ordinary map location range. */
 loc_range_t macro; /* Macro map location range.  */
-int ordinary_delta;/* Add to ordinary loc to get serialized loc.  
*/
-int macro_delta;   /* Likewise for macro loc.  */
+/* Add to locs to get serialized loc.  */
+location_diff_t ordinary_delta;
+location_diff_t macro_delta;
   };
 
 private:
@@ -3304,9 +3330,9 @@ static loc_spans spans;
 struct ord_loc_info
 {
   const line_map_ordinary *src; // line map we're based on
-  unsigned offset; // offset to this line
-  unsigned span;   // number of locs we span
-  unsigned remap;  // serialization
+  line_map_uint_t offset;   // offset to 

[PATCH 04/15] tree-phinodes: Use 4 instead of 2 as the minimum number of phi args

[Lewis Hyatt]

Currently, when we allocate a gphi object, we round up the capacity for the
trailing arguments array such that it will make full use of the page size
that ggc will allocate. While there is also an explicit minimum of 2
arguments, in practice after rounding to the ggc page size there is always
room for at least 4.

It seems we have some code that has come to depend on there being this much
room before reallocation of a PHI is required. For example, the function
loop_version () used during loop optimization will make sure there is room
for an additional edge on each PHI that it processes. But there are call
sites which cache a PHI pointer prior to calling loop_version () and assume
it remains valid afterward, thus implicitly assuming that the PHI will have
spare capacity. Examples include split_loop () and gen_parallel_loop ().

This works fine now, but if the size of a gphi becomes larger, e.g. due to
configuring location_t to be a 64-bit type, then on 32-bit platforms it ends
up being possible to get a gphi with only 2 arguments of capacity, causing
the above call sites of loop_version () to fail. (They store a pointer to a
gphi object that no longer has the same meaning it did before it got
reallocated.) The testcases gcc.dg/torture/pr113707-2.c and
gcc.dg/graphite/pr81945.c exhibit that failure mode.

It may be necessary to adjust those call sites to make this more robust, but
in the meantime, changing the minimum from 2 to 4 does no harm given the
minimum is practically 4 anyway, and it resolves the issue for 32-bit
platforms.

gcc/ChangeLog:

* tree-phinodes.cc (MIN_PHI_ARGS): New constant.
(allocate_phi_node): Change from hard-coded value 2 to MIN_PHI_ARGS,
which is now 4.
(ideal_phi_node_len): Likewise.
(release_phi_node): Likewise.
---
 gcc/tree-phinodes.cc | 24 
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/gcc/tree-phinodes.cc b/gcc/tree-phinodes.cc
index 5a7e4a94e57..9d8e16ac200 100644
--- a/gcc/tree-phinodes.cc
+++ b/gcc/tree-phinodes.cc
@@ -63,11 +63,12 @@ along with GCC; see the file COPYING3.  If not see
walking the elements of the last array entry would result in finding less
than .1% additional reusable PHI nodes.
 
-   Note that we can never have less than two PHI argument slots.  Thus,
-   the -2 on all the calculations below.  */
+   Note that we can never have less than MIN_PHI_ARGS argument slots.  Thus,
+   the subtraction of MIN_PHI_ARGS on all the calculations below.  */
 
 #define NUM_BUCKETS 10
-static GTY ((deletable (""))) vec *free_phinodes[NUM_BUCKETS 
- 2];
+#define MIN_PHI_ARGS 4
+static GTY ((deletable (""))) vec *free_phinodes[NUM_BUCKETS 
- MIN_PHI_ARGS];
 static unsigned long free_phinode_count;
 
 static int ideal_phi_node_len (int);
@@ -94,17 +95,18 @@ static inline gphi *
 allocate_phi_node (size_t len)
 {
   gphi *phi;
-  size_t bucket = NUM_BUCKETS - 2;
+  size_t bucket = NUM_BUCKETS - MIN_PHI_ARGS;
   size_t size = sizeof (struct gphi)
+ (len - 1) * sizeof (struct phi_arg_d);
 
   if (free_phinode_count)
-for (bucket = len - 2; bucket < NUM_BUCKETS - 2; bucket++)
+for (bucket = len - MIN_PHI_ARGS; bucket < NUM_BUCKETS - MIN_PHI_ARGS;
+bucket++)
   if (free_phinodes[bucket])
break;
 
   /* If our free list has an element, then use it.  */
-  if (bucket < NUM_BUCKETS - 2
+  if (bucket < NUM_BUCKETS - MIN_PHI_ARGS
   && gimple_phi_capacity ((*free_phinodes[bucket])[0]) >= len)
 {
   free_phinode_count--;
@@ -145,9 +147,8 @@ ideal_phi_node_len (int len)
   size_t size, new_size;
   int log2, new_len;
 
-  /* We do not support allocations of less than two PHI argument slots.  */
-  if (len < 2)
-len = 2;
+  /* We do not support allocations of less than MIN_PHI_ARGS argument slots.  
*/
+  len = MAX (len, MIN_PHI_ARGS);
 
   /* Compute the number of bytes of the original request.  */
   size = sizeof (struct gphi)
@@ -225,14 +226,13 @@ release_phi_node (gimple *phi)
 
   /* Immediately return the memory to the allocator when we would
  only ever re-use it for a smaller size allocation.  */
-  if (len - 2 >= NUM_BUCKETS - 2)
+  if (len >= NUM_BUCKETS)
 {
   ggc_free (phi);
   return;
 }
 
-  bucket = len > NUM_BUCKETS - 1 ? NUM_BUCKETS - 1 : len;
-  bucket -= 2;
+  bucket = len - MIN_PHI_ARGS;
   vec_safe_push (free_phinodes[bucket], phi);
   free_phinode_count++;
 }

[PATCH 14/15] Support for 64-bit location_t: Testsuite parts

[Lewis Hyatt]

Add support to the testsuite for effective target "large_location_t"
indicating if 64-bit location support has been configured. Adjust the tests
that are sensitive to location_t internals so they can test large locations
too.

gcc/testsuite/ChangeLog:

* lib/target-supports.exp (check_no_compiler_messages_nocache):
Refactor the code for retrieving compiler messages to...
(get_compiler_messages_nocache) ...here. New function.
(get_compiler_messages): New function.
(check_effective_target_large_location_t): New function.
* g++.dg/diagnostic/pr77949.C: Adapt the test for 64-bit location_t,
when the expected failure doesn't actually happen.
* g++.dg/modules/loc-prune-4.C: Adjust the expected output for the
64-bit location_t case
* gcc.dg/plugin/expensive_selftests_plugin.c: Don't try to test
the maximum supported column number in 64-bit location_t mode.
* gcc.dg/plugin/location_overflow_plugin.c: Adjust the base_location
so it can effectively test 64-bit location_t as well.
---
 gcc/testsuite/g++.dg/diagnostic/pr77949.C |  5 ++-
 gcc/testsuite/g++.dg/modules/loc-prune-4.C| 11 --
 .../plugin/expensive_selftests_plugin.c   |  7 ++--
 .../gcc.dg/plugin/location_overflow_plugin.c  |  7 
 gcc/testsuite/lib/target-supports.exp | 34 +--
 5 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/gcc/testsuite/g++.dg/diagnostic/pr77949.C 
b/gcc/testsuite/g++.dg/diagnostic/pr77949.C
index b81d6e2bb46..35922e9bb23 100644
--- a/gcc/testsuite/g++.dg/diagnostic/pr77949.C
+++ b/gcc/testsuite/g++.dg/diagnostic/pr77949.C
@@ -1,7 +1,10 @@
-// Ensure that no fix-it hints are emitted
+// Ensure that no fix-it hints are emitted unless large locations are 
available.
 // { dg-options "-fdiagnostics-parseable-fixits" }
 
 /* Very long line, where a missing semicolon would be suggested for
insertion at column 4097.  */
 class test {   




































 

[PATCH 01/15] Support for 64-bit location_t: libcpp parts

[Lewis Hyatt]

This patch adds support in the libcpp line-maps infrastructure for a 64-bit
location_t type that is capable of representing more source locations than
the current 32-bit location_t. The support will be made configurable at
build time in a subsequent patch; for now it will be off by default.

libcpp/ChangeLog:

* include/line-map.h (location_t): Conditionally make this typedef a
64-bit integer if so configured, and adjust related constants to
match.
(line_map_uint_t): New typedef, the same type as location_t.
(struct line_map): Adjust comments about the struct layout.
(struct line_map_macro): Likewise.
(struct line_map_ordinary): Likewise, and also reorder the fields to
avoid padding in case 64-bit location was configured.
(struct maps_info_ordinary): Change member types from "unsigned int"
to "line_map_uint_t".
(struct maps_info_macro): Likewise.
(struct location_adhoc_data_map): Likewise.
(LINEMAPS_ALLOCATED): Change return type from "unsigned int" to
"line_map_uint_t".
(LINEMAPS_ORDINARY_ALLOCATED): Likewise.
(LINEMAPS_MACRO_ALLOCATED): Likewise.
(LINEMAPS_USED): Likewise.
(LINEMAPS_ORDINARY_USED): Likewise.
(LINEMAPS_MACRO_USED): Likewise.
(linemap_lookup_macro_index): Likewise.
(LINEMAPS_MAP_AT): Change argument type from "unsigned int" to
"line_map_uint_t".
(LINEMAPS_ORDINARY_MAP_AT): Likewise.
(LINEMAPS_MACRO_MAP_AT): Likewise.
(line_map_new_raw): Likewise.
(linemap_module_restore): Likewise.
(linemap_dump): Likewise.
(line_table_dump): Likewise.
(LINEMAPS_LAST_MAP): Add a linemap_assert() for safety.
(SOURCE_COLUMN): Handle 64-bit location_t.
* line-map.cc (location_adhoc_data_hash): Don't truncate locations
to 32-bit prematurely.
(line_maps::get_or_create_combined_loc): Adapt types to support
64-bit location_t. Use MAX_LOCATION_T rather than a hard-coded
constant.
(line_maps::get_range_from_loc): Adapt types and constants to
support 64-bit location_t.
(line_maps::pure_location_p): Likewise.
(line_maps::get_pure_location): Likewise.
(line_map_new_raw): Likewise.
(LAST_SOURCE_LINE_LOCATION): Likewise.
(linemap_add): Likewise.
(linemap_module_restore): Likewise.
(linemap_line_start): Likewise.
(linemap_position_for_column): Likewise.
(linemap_position_for_line_and_column): Likewise.
(linemap_position_for_loc_and_offset): Likewise.
(linemap_ordinary_map_lookup): Likewise.
(linemap_lookup_macro_index): Likewise.
(linemap_dump): Likewise.
(linemap_dump_location): Likewise.
(linemap_get_file_highest_location): Likewise.
(line_table_dump): Likewise.
(linemap_compare_locations): Avoid signed int overflow in the result.
* include/cpplib.h (struct cpp_identifier): Update comment about the
struct layout.
* macro.cc (num_expanded_macros_counter): Change type of global
variable from "unsigned int" to "line_map_uint_t".
(num_macro_tokens_counter): Likewise.
---
 libcpp/include/cpplib.h   |   3 +-
 libcpp/include/line-map.h | 139 +-
 libcpp/line-map.cc| 138 +
 libcpp/macro.cc   |   4 +-
 4 files changed, 176 insertions(+), 108 deletions(-)

diff --git a/libcpp/include/cpplib.h b/libcpp/include/cpplib.h
index 267d28147ab..f4913ac4313 100644
--- a/libcpp/include/cpplib.h
+++ b/libcpp/include/cpplib.h
@@ -256,7 +256,8 @@ struct GTY(()) cpp_identifier {
 };
 
 /* A preprocessing token.  This has been carefully packed and should
-   occupy 16 bytes on 32-bit hosts and 24 bytes on 64-bit hosts.  */
+   occupy 16 bytes on 32-bit hosts and 24 bytes on 64-bit hosts.  With
+   64-bit locations enabled, it occupies rather 32 bytes.  */
 struct GTY(()) cpp_token {
 
   /* Location of first char of token, together with range of full token.  */
diff --git a/libcpp/include/line-map.h b/libcpp/include/line-map.h
index 732ec5e6445..65610b5a2ef 100644
--- a/libcpp/include/line-map.h
+++ b/libcpp/include/line-map.h
@@ -291,7 +291,12 @@ enum lc_reason
 
To further see how location_t works in practice, see the
worked example in libcpp/location-example.txt.  */
+
+#ifndef ENABLE_LARGE_SOURCE_LOCATIONS
+
+static constexpr bool LINE_MAP_LARGE_MODE = false;
 typedef unsigned int location_t;
+typedef int location_diff_t;
 
 /* Do not track column numbers higher than this one.  As a result, the
range of column_bits is [12, 18] (or 0 if column numbers are
@@ -311,6 +316,43 @@ const location_t LINE_MAP_MAX_LOCATION_WITH_COLS = 
0x6000;
 /* Highest possible source location encoded within an ordinary map.  */
 const location_t LINE_MAP_MAX_LOCATION = 0x7000;
 
+/* This is the

[PATCH 09/15] Support for 64-bit location_t: Frontend parts

[Lewis Hyatt]

The C/C++ frontend code contains a couple instances where a callback
receiving a "location_t" argument is prototyped to take "unsigned int"
instead. This will make a difference once location_t can be configured to a
different type, so adjust that now.

gcc/c-family/ChangeLog:

* c-lex.cc (cb_ident): Change "unsigned int" argument to type
"location_t".
(cb_def_pragma): Likewise.
(cb_define): Likewise.
(cb_undef): Likewise.
---
 gcc/c-family/c-lex.cc | 10 +-
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/gcc/c-family/c-lex.cc b/gcc/c-family/c-lex.cc
index f7168ce6fc0..b14f942b87b 100644
--- a/gcc/c-family/c-lex.cc
+++ b/gcc/c-family/c-lex.cc
@@ -54,10 +54,10 @@ static tree lex_charconst (const cpp_token *);
 static void update_header_times (const char *);
 static int dump_one_header (splay_tree_node, void *);
 static void cb_line_change (cpp_reader *, const cpp_token *, int);
-static void cb_ident (cpp_reader *, unsigned int, const cpp_string *);
-static void cb_def_pragma (cpp_reader *, unsigned int);
-static void cb_define (cpp_reader *, unsigned int, cpp_hashnode *);
-static void cb_undef (cpp_reader *, unsigned int, cpp_hashnode *);
+static void cb_ident (cpp_reader *, location_t, const cpp_string *);
+static void cb_def_pragma (cpp_reader *, location_t);
+static void cb_define (cpp_reader *, location_t, cpp_hashnode *);
+static void cb_undef (cpp_reader *, location_t, cpp_hashnode *);
 
 void
 init_c_lex (void)
@@ -164,7 +164,7 @@ dump_time_statistics (void)
 
 static void
 cb_ident (cpp_reader * ARG_UNUSED (pfile),
- unsigned int ARG_UNUSED (line),
+ location_t ARG_UNUSED (line),
  const cpp_string * ARG_UNUSED (str))
 {
   if (!flag_no_ident)

[PATCH 12/15] Support for 64-bit location_t: Backend parts

[Lewis Hyatt]

A few targets have been using "unsigned int" function arguments that need to
receive a "location_t". Change to "location_t" to prepare for the
possibility that location_t can be configured to be a different type.

gcc/ChangeLog:

* config/aarch64/aarch64-c.cc (aarch64_resolve_overloaded_builtin):
Change "unsigned int" argument to "location_t".
* config/avr/avr-c.cc (avr_resolve_overloaded_builtin): Likewise.
* config/riscv/riscv-c.cc (riscv_resolve_overloaded_builtin): Likewise.
* target.def: Likewise.
* doc/tm.texi: Regenerate.
---
 gcc/config/aarch64/aarch64-c.cc | 3 +--
 gcc/config/avr/avr-c.cc | 3 +--
 gcc/config/riscv/riscv-c.cc | 3 +--
 gcc/doc/tm.texi | 2 +-
 gcc/target.def  | 2 +-
 5 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-c.cc b/gcc/config/aarch64/aarch64-c.cc
index f9b9e379375..fff10337fb3 100644
--- a/gcc/config/aarch64/aarch64-c.cc
+++ b/gcc/config/aarch64/aarch64-c.cc
@@ -365,11 +365,10 @@ aarch64_pragma_aarch64 (cpp_reader *)
 
 /* Implement TARGET_RESOLVE_OVERLOADED_BUILTIN.  */
 static tree
-aarch64_resolve_overloaded_builtin (unsigned int uncast_location,
+aarch64_resolve_overloaded_builtin (location_t location,
tree fndecl, void *uncast_arglist)
 {
   vec empty = {};
-  location_t location = (location_t) uncast_location;
   vec *arglist = (uncast_arglist
   ? (vec *) uncast_arglist
   : &empty);
diff --git a/gcc/config/avr/avr-c.cc b/gcc/config/avr/avr-c.cc
index d3c40d73043..7cf8344c1c7 100644
--- a/gcc/config/avr/avr-c.cc
+++ b/gcc/config/avr/avr-c.cc
@@ -48,11 +48,10 @@ enum avr_builtin_id
 /* Implement `TARGET_RESOLVE_OVERLOADED_PLUGIN'.  */
 
 static tree
-avr_resolve_overloaded_builtin (unsigned int iloc, tree fndecl, void *vargs)
+avr_resolve_overloaded_builtin (location_t loc, tree fndecl, void *vargs)
 {
   tree type0, type1, fold = NULL_TREE;
   avr_builtin_id id = AVR_BUILTIN_COUNT;
-  location_t loc = (location_t) iloc;
   vec &args = * (vec*) vargs;
 
   switch (DECL_MD_FUNCTION_CODE (fndecl))
diff --git a/gcc/config/riscv/riscv-c.cc b/gcc/config/riscv/riscv-c.cc
index c59f408d3a8..7f78e2cf019 100644
--- a/gcc/config/riscv/riscv-c.cc
+++ b/gcc/config/riscv/riscv-c.cc
@@ -312,11 +312,10 @@ riscv_check_builtin_call (location_t loc, vec 
arg_loc, tree fndecl,
 
 /* Implement TARGET_RESOLVE_OVERLOADED_BUILTIN.  */
 static tree
-riscv_resolve_overloaded_builtin (unsigned int uncast_location, tree fndecl,
+riscv_resolve_overloaded_builtin (location_t loc, tree fndecl,
  void *uncast_arglist)
 {
   vec empty = {};
-  location_t loc = (location_t) uncast_location;
   vec *arglist = (vec *) uncast_arglist;
   unsigned int code = DECL_MD_FUNCTION_CODE (fndecl);
   unsigned int subcode = code >> RISCV_BUILTIN_SHIFT;
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index 4deb3d2c283..4b739578f4f 100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -12108,7 +12108,7 @@ ignored.  This function should return the result of the 
call to the
 built-in function.
 @end deftypefn
 
-@deftypefn {Target Hook} tree TARGET_RESOLVE_OVERLOADED_BUILTIN (unsigned int 
@var{loc}, tree @var{fndecl}, void *@var{arglist})
+@deftypefn {Target Hook} tree TARGET_RESOLVE_OVERLOADED_BUILTIN (location_t 
@var{loc}, tree @var{fndecl}, void *@var{arglist})
 Select a replacement for a machine specific built-in function that
 was set up by @samp{TARGET_INIT_BUILTINS}.  This is done
 @emph{before} regular type checking, and so allows the target to
diff --git a/gcc/target.def b/gcc/target.def
index 523ae7ec9aa..e285cef5743 100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -2497,7 +2497,7 @@ arguments passed to the built-in function.  The result is 
a\n\
 complete expression that implements the operation, usually\n\
 another @code{CALL_EXPR}.\n\
 @var{arglist} really has type @samp{VEC(tree,gc)*}",
- tree, (unsigned int /*location_t*/ loc, tree fndecl, void *arglist), NULL)
+ tree, (location_t loc, tree fndecl, void *arglist), NULL)
 
 DEFHOOK
 (check_builtin_call,

[PATCH 06/15] gimple: Handle tail padding when computing gimple_ops_offset

[Lewis Hyatt]

The array gimple_ops_offset_[], which is used to find the trailing op[]
array for a given gimple struct, is computed assuming that op[] will be
found at sizeof(tree) bytes away from the end of the struct. This is only
correct if the alignment requirement of a pointer is the same as the
alignment requirement of the struct, otherwise there will be padding bytes
that invalidate the calculation. On 64-bit platforms, this generally works
fine because a pointer has 8-byte alignment and none of the structs make use
of more than that. On 32-bit platforms, it also currently works fine because
there are no 64-bit integers in the gimple structs. There are 32-bit
platforms (e.g. sparc) on which a pointer has 4-byte alignment and a
uint64_t has 8-byte alignment. On such platforms, adding a uint64_t to the
gimple structs (such as a location_t with --enable-large-source-locations)
causes gimple_ops_offset_ to be 4 bytes too large.

It would be nice to use offsetof() to compute the offset exactly, but
offsetof() is not guaranteed to work for these types, because they use
inheritance and so are not standard layout types. This patch attempts to
detect the presence of tail padding by detecting when such padding is reused
by inheritance; the padding should generally be reused for the same reason
that offsetof() is not available, namely that all the relevant types use
inheritance. One could envision systems on which this fix does not go far
enough (e.g., if the ABI forbids reuse of tail padding), but it makes things
better without affecting anything that currently works.

gcc/ChangeLog:

* gimple.cc (get_tail_padding_adjustment): New function.
(DEFGSSTRUCT): Adjust the computation of gimple_ops_offset_ to be
correct in the presence of tail padding.
---
 gcc/gimple.cc | 34 +-
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/gcc/gimple.cc b/gcc/gimple.cc
index f7b313be40e..f0a642f5b51 100644
--- a/gcc/gimple.cc
+++ b/gcc/gimple.cc
@@ -52,12 +52,36 @@ along with GCC; see the file COPYING3.  If not see
 #include "ipa-modref.h"
 #include "dbgcnt.h"
 
-/* All the tuples have their operand vector (if present) at the very bottom
-   of the structure.  Therefore, the offset required to find the
-   operands vector the size of the structure minus the size of the 1
-   element tree array at the end (see gimple_ops).  */
+/* All the tuples have their operand vector (if present) at the very bottom of
+   the structure.  Therefore, the offset required to find the operands vector 
is
+   the size of the structure minus the size of the 1-element tree array at the
+   end (see gimple_ops).  An adjustment may be required if there is tail
+   padding, as may happen on a host (e.g. sparc) where a pointer has 4-byte
+   alignment while a uint64_t has 8-byte alignment.
+
+   Unfortunately, we can't use offsetof to do this computation 100%
+   straightforwardly, because these structs use inheritance and so are not
+   standard layout types.  However, the fact that they are not standard layout
+   types also means that tail padding will be reused in inheritance, which 
makes
+   it possible to check for the problematic case with the following logic
+   instead.  If tail padding is detected, the offset should be decreased
+   accordingly.  */
+
+template
+static constexpr size_t
+get_tail_padding_adjustment ()
+{
+  struct padding_check : G
+  {
+tree t;
+  };
+  return sizeof (padding_check) == sizeof (G) ? sizeof (tree) : 0;
+}
+
 #define DEFGSSTRUCT(SYM, STRUCT, HAS_TREE_OP) \
-   (HAS_TREE_OP ? sizeof (struct STRUCT) - sizeof (tree) : 0),
+  (HAS_TREE_OP \
+   ? sizeof (STRUCT) - sizeof (tree) - get_tail_padding_adjustment () \
+   : 0),
 EXPORTED_CONST size_t gimple_ops_offset_[] = {
 #include "gsstruct.def"
 };

[PATCH 05/15] c++: Fix tree_contains_struct for TRAIT_EXPR

[Lewis Hyatt]

CODE_CONTAINS_STRUCT () currently reports that a TRAIT_EXPR contains a
TS_EXP struct, but it does not actually start with a TS_EXP as an initial
sequence. In modules.cc, when we stream out a tree, we explicitly check for the
TS_EXP case and call note_location(t->exp.locus) if so. Currently, this
actually queries the tree_common::chain field of a tree_trait_expr, which
seems not to be used, returning 0, which is interpreted as UNKNOWN_LOCATION
and does no harm.

If location_t has been configured to be 64 bytes, then on 32-bit platforms
(well those, such as sparc, on which uint64_t has higher alignment requirement
than a pointer), reading t->exp.locus ends up reading a different field
(tree_trait_expr::type1) due to padding offsets. That field is not generally
0, and the resulting bogus location_t is sufficiently problematic to cause
an ICE in the line_map code. Pretty much any modules testcase displays the
issue, such as partial-2_a.C.

Resolve by initializing tree_contains_struct with the correct value for
TRAIT_EXPR, namely TS_TYPED.

gcc/cp/ChangeLog:

* cp-objcp-common.cc (cp_common_init_ts): Change TRAIT_EXPR from
TS_EXP to TS_TYPED.
---
 gcc/cp/cp-objcp-common.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcc/cp/cp-objcp-common.cc b/gcc/cp/cp-objcp-common.cc
index 7a0636f1653..1e43db31db8 100644
--- a/gcc/cp/cp-objcp-common.cc
+++ b/gcc/cp/cp-objcp-common.cc
@@ -617,6 +617,7 @@ cp_common_init_ts (void)
   MARK_TS_TYPED (PTRMEM_CST);
   MARK_TS_TYPED (LAMBDA_EXPR);
   MARK_TS_TYPED (TYPE_ARGUMENT_PACK);
+  MARK_TS_TYPED (TRAIT_EXPR);
 
   /* Random new trees.  */
   MARK_TS_COMMON (BASELINK);
@@ -684,7 +685,6 @@ cp_common_init_ts (void)
   MARK_TS_EXP (TAG_DEFN);
   MARK_TS_EXP (TEMPLATE_ID_EXPR);
   MARK_TS_EXP (THROW_EXPR);
-  MARK_TS_EXP (TRAIT_EXPR);
   MARK_TS_EXP (TYPEID_EXPR);
   MARK_TS_EXP (TYPE_EXPR);
   MARK_TS_EXP (UNARY_PLUS_EXPR);

[PATCH 08/15] Support for 64-bit location_t: Analyzer parts

[Lewis Hyatt]

The analyzer occasionally prints internal location_t values for debugging;
adjust those parts so they will work if location_t is 64-bit. For
simplicity, to avoid hassling with the printf format string, just convert to
(unsigned long long) in either case.

gcc/analyzer/ChangeLog:

* checker-event.cc (checker_event::dump): Support printing either
32- or 64-bit location_t values.
* checker-path.cc (checker_path::inject_any_inlined_call_events):
Likewise.
---
 gcc/analyzer/checker-event.cc | 4 ++--
 gcc/analyzer/checker-path.cc  | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/gcc/analyzer/checker-event.cc b/gcc/analyzer/checker-event.cc
index 5a292377e93..bb26f71e4b4 100644
--- a/gcc/analyzer/checker-event.cc
+++ b/gcc/analyzer/checker-event.cc
@@ -188,8 +188,8 @@ checker_event::dump (pretty_printer *pp) const
   if (m_effective_fndecl != m_original_fndecl)
pp_printf (pp, " corrected from %qE", m_original_fndecl);
 }
-  pp_printf (pp, ", m_loc=%x)",
-get_location ());
+  pp_printf (pp, ", m_loc=%llx)",
+(unsigned long long) get_location ());
 }
 
 /* Dump this event to stderr (for debugging/logging purposes).  */
diff --git a/gcc/analyzer/checker-path.cc b/gcc/analyzer/checker-path.cc
index 98b59884174..25d95fe0bfc 100644
--- a/gcc/analyzer/checker-path.cc
+++ b/gcc/analyzer/checker-path.cc
@@ -281,8 +281,9 @@ checker_path::inject_any_inlined_call_events (logger 
*logger)
  logger->log_partial ("  %qE", iter.get_block ());
  if (!flag_dump_noaddr)
logger->log_partial (" (%p)", iter.get_block ());
- logger->log_partial (", fndecl: %qE, callsite: 0x%x",
-  iter.get_fndecl (), iter.get_callsite ());
+ logger->log_partial (", fndecl: %qE, callsite: 0x%llx",
+  iter.get_fndecl (),
+  (unsigned long long) iter.get_callsite ());
  if (iter.get_callsite ())
dump_location (logger->get_printer (), iter.get_callsite ());
  logger->end_log_line ();

[PATCH 03/15] tree-cfg: Fix call to next_discriminator_for_locus()

[Lewis Hyatt]

While testing 64-bit location_t support, I ran into an -fcompare-debug issue
that was traced back here. Despite the name, next_discriminator_for_locus()
is meant to take an integer line number argument, not a location_t. There is
one call site which has been passing a location_t instead. For the most part
that is harmless, although in case there are two CALL stmts on the same line
with different location_t, it may fail to generate a unique discriminator
where it should. Once location_t is configured to be 64-bit, however, it
produces an -fcompare-debug failure which is what I noticed. Fix it by passing
the line number rather than the location_t.

I am not aware of a testcase that demonstrates any observable wrong
behavior, but the file debug/pr53466.C is an example where the discriminator
assignment is indeed different before and after this change.

gcc/ChangeLog:

* tree-cfg.cc (assign_discriminators): Fix incorrect value passed to
next_discriminator_for_locus().
---
 gcc/tree-cfg.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index 3eede0d61cd..c2100a51a7a 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -1251,7 +1251,7 @@ assign_discriminators (void)
}
  /* Allocate a new discriminator for CALL stmt.  */
  if (gimple_code (stmt) == GIMPLE_CALL)
-   curr_discr = next_discriminator_for_locus (curr_locus);
+   curr_discr = next_discriminator_for_locus (curr_locus_e.line);
}
 
   gimple *last = last_nondebug_stmt (bb);

[PATCH 02/15] libcpp: Fix potential unaligned access in cpp_buffer

[Lewis Hyatt]

libcpp makes use of the cpp_buffer pfile->a_buff to store things while it is
handling macros. It uses it to store pointers (cpp_hashnode*, for macro
arguments) and cpp_macro objects. This works fine because a cpp_hashnode*
and a cpp_macro have the same alignment requirement on either 32-bit or
64-bit systems (namely, the same alignment as a pointer.)

When 64-bit location_t is enabled on a 32-bit sytem, the alignment
requirement may cease to be the same, because the alignment requirement of a
cpp_macro object changes to that of a uint64_t, which be larger than that of
a pointer. It's not the case for x86 32-bit, but for example, on sparc, a
pointer has 4-byte alignment while a uint64_t has 8. In that case,
intermixing the two within the same cpp_buffer leads to a misaligned
access. The code path that triggers this is the one in _cpp_commit_buff in
which a hash table with its own allocator (i.e. ggc) is not being used, so
it doesn't happen within the compiler itself, but it happens in the other
libcpp clients, such as genmatch.

Fix that up by ensuring _cpp_commit_buff commits a fully aligned chunk of the
buffer, so it's ready for anything it may be used for next.

For good measure, also modify CPP_ALIGN so that it guarantees to return an
alignment at least the size of location_t. Currently it returns the max of
a pointer and a double. I am not aware of any platform where a double may
have smaller alignment than a uint64_t, but it does not hurt to add
location_t here to be sure.

libcpp/ChangeLog:

* lex.cc (_cpp_commit_buff): Make sure that the buffer is properly
aligned for the next allocation.
* internal.h (struct dummy): Make sure alignment is large enough for
a location_t, just in case.
---
 libcpp/internal.h |  1 +
 libcpp/lex.cc | 10 --
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/libcpp/internal.h b/libcpp/internal.h
index e65198e89da..358e77cd622 100644
--- a/libcpp/internal.h
+++ b/libcpp/internal.h
@@ -85,6 +85,7 @@ struct dummy
   {
 double d;
 int *p;
+location_t l;
   } u;
 };
 
diff --git a/libcpp/lex.cc b/libcpp/lex.cc
index 849447eb4d7..858970b5d17 100644
--- a/libcpp/lex.cc
+++ b/libcpp/lex.cc
@@ -4997,7 +4997,8 @@ _cpp_aligned_alloc (cpp_reader *pfile, size_t len)
 void *
 _cpp_commit_buff (cpp_reader *pfile, size_t size)
 {
-  void *ptr = BUFF_FRONT (pfile->a_buff);
+  const auto buff = pfile->a_buff;
+  void *ptr = BUFF_FRONT (buff);
 
   if (pfile->hash_table->alloc_subobject)
 {
@@ -5006,7 +5007,12 @@ _cpp_commit_buff (cpp_reader *pfile, size_t size)
   ptr = copy;
 }
   else
-BUFF_FRONT (pfile->a_buff) += size;
+{
+  BUFF_FRONT (buff) += size;
+  /* Make sure the remaining space is maximally aligned for whatever this
+buffer holds next.  */
+  BUFF_FRONT (buff) += BUFF_ROOM (buff) % DEFAULT_ALIGNMENT;
+}
 
   return ptr;
 }

[PATCH 00/15] Support for 64-bit location_t

[Lewis Hyatt]

Hello-

There is no shortage of PRs complaining about things that go wrong when the
line_maps data structure in libcpp starts to run into its limits. Being
restricted to a 32-bit location_t to cover all source locations means it has
the following limitations, among others:

-Column numbers larger than 2^12 are not tracked at all, so diagnostics
 can only refer to the whole line, and things like -Wmisleading-indentation
 stop working with large column numbers. [PR89549]

-Tokens longer than 32 characters cannot be stored compactly in the line
 map and have to be stored as ad-hoc locations with the associated
 memory and locality overhead.

-Sufficiently large sources can exhaust the available location_t space.

The current limitations are not encountered too much, but it does happen
often enough, especially in testsuites or large PCH compiles that include,
e.g., every header in a large project.

Currently, the main workaround is to use the option -flarge-source-locations.
This sets the range bits in the line_maps to 0, effectively meaning that
every source code token is stored as an ad-hoc location. This does
significantly increase the number of locations that can be stored, but it
loses all the benefits of the line_maps infrastructure, and it does not help
with the 4096 limit for column numbers.

It seemed to me like there's no way to improve on that other than letting
location_t be 64 bits instead of 32 bits. I thought I would take a look at
what that would entail... the attached 15 (mostly small) patches are the
result. It was unfortunately necessary to touch a lot of different parts of
the compiler, since location_t is used pretty pervasively. This patch set
adds a configure option --enable-large-source-locations that makes
location_t be 64 bits. The option is off by default.

There are decisions to be made about what to do with all those new bits. I
made some choices in this patch, but it's easy to change if different
tradeoffs are preferable. Here is what I did:

- Only 63 bits are used, the high bit is not used so that two location_t
  can be safely subtracted and stored in an int64_t.

- As in the 32-bit case, half of the available locations are reserved
  for ad-hoc locations, and the other half for macros and ordinary
  locations.

- The limit on the maximum column number is removed, so it's now set to
  2^31-1. (GCC right now pervasively assumes line and column numbers
  within a file can be represented by a 32-bit signed int. That could be
  changed too, but it is not part of this patch and would be another
  similarly large change.)

- The default range bits is changed from 5 to 7. This means that
  tokens up to 128 chars in length get ordinary locations, while longer
  ones get ad-hoc locations. I think that's a big improvement from the
  current max of 32 and will result in a lot fewer ad-hoc locations
  being generated. There are probably enough bits to go around, though,
  that this could be made even larger too.

All that stuff is configured in line-map.h and is easy to change. The bulk
of the patch is dealing with the fallout from making location_t's type
configurable; in particular, C++ modules needed a lot of tweaks, since they
deal with line_map internals, and I had to add a new field to RTL which
necessitated a lot of small changes. I was not previously familiar with
RTL... I think I generally grokked the idea there and I think the changes I
made are safe, but I am pretty sure some of them are unnecessary so would
appreciate feedback on that too.

I tried to test on a variety of platforms. For each of them, I did three
bootstrap + regtest (without the patch, with the patch and without
--enable-large-source-locations, and with the patch and with
--enable-large-source-locations) with --enable-checking=yes,extra,rtl and
verified no change in the testsuite other than the expected new PASSes. Here
is what I have tested so far:

all languages: x86_64, aarch64 (cfarm185), ppc64le (cfarm135)
c/c++ only: sparc 32-bit (cfarm216)

The testing on sparc 32-bit was very productive and in particular identified
a few issues with padding and alignment that have not been apparent until
now. That platform differs from x86 32-bit in that uint64_t has a larger
alignment than a pointer does.

Also it's worth noting, even if there is no interest in supporting this
feature, patches 2, 3, 4, 5, and 6 I think are worth accepting regardless;
these are small bug fixes that came up in testing and are independent of
whether location_t is changed or not... they could be triggered by other
changes in the future too.

Regarding memory usage implications of this change. It does make a lot of
data structures larger. Here is what it does on x86-64. The base sizes of some
types change as follows:

location_t:  From 4 to 8 (+100%)
line_map_ordinary:   From 24 to 32 (+33%)
line_map_macro:  From 32 to 40 (+25%

Re: [PATCH] diagnostics: libcpp: Improve locations for _Pragma lexing diagnostics [PR114423]

[Lewis Hyatt]

On Fri, Oct 18, 2024 at 6:21 PM David Malcolm  wrote:
>
> On Fri, 2024-10-18 at 13:58 -0400, Lewis Hyatt wrote:
> > On Fri, Oct 18, 2024 at 11:25 AM David Malcolm 
> > wrote:
> > > >if (!pfile->cb.diagnostic)
> > > >  abort ();
> > > > -  ret = pfile->cb.diagnostic (pfile, level, reason, richloc,
> > > > _(msgid), ap);
> > > > -
> > > > -  return ret;
> > > > +  if (pfile->diagnostic_override_loc && level != CPP_DL_NOTE)
> > > > +{
> > > > +  rich_location rc2{pfile->line_table, pfile-
> > > > > diagnostic_override_loc};
> > > > +  return pfile->cb.diagnostic (pfile, level, reason, &rc2,
> > > > _(msgid), ap);
> > > > +}
> > >
> > > This will effectively override the primary location in the
> > > rich_location, but by using a second rich_location instance it will
> > > also ignore any secondary locations and fix-it hints.
> > >
> > > This might will be what we want here, but did you consider
> > >   richloc.set_range (0, pfile->diagnostic_override_loc,
> > >  SHOW_RANGE_WITH_CARET);
> > > to reset the primary location?
> > >
> > > Otherwise, looks good to me.
> > >
> > > [...snip...]
> > >
> > > Thanks
> > > Dave
> > >
> >
> > Thanks for taking a look! My thinking was, when libcpp produces
> > tokens
> > from a _Pragma string, basically every location_t that it generates
> > is
> > wrong and shouldn't be used. Because it doesn't actually set up the
> > line_map, it gets something random that's just sorta close to
> > reasonable. So I think it makes sense to discard fixits and secondary
> > locations too.
>
> Fair enough.  I think the patch is OK, then, assuming usual testing.

...

> > For this particular case I could improve it with a one line addition
> > like
> >
> > rc2.set_escape_on_output (richloc->escape_on_output_p ());
> >
> > and that would actually handle all currently needed cases since we
> > don't use a lot of rich_locations in libcpp. It would just become
> > stale if some other option gets added to rich_location in the future
> > that we also should preserve.
> > I think to fix it in a fully general
> > way, it would be necessary to add a new interface to class
> > rich_location. It already has a method to delete all the fixit hints,
> > it would also need a method to delete all the ranges. Then we could
> > make a copy of the richloc and just delete everything we don't want
> > to
> > preserve. Do you have a preference one way or the other?
>
> Do we know which diagnostics are likely to be emitted when we override
> the location?  I suspect that anywhere that's overriding the location
> is an awkward case where we're unlikely to have fix-it hints and
> secondary ranges.
>
> I don't have a strong preference here.
>
> >
> > By the way, your suggestion was to directly modify richloc... these
> > functions do take the richloc by non-const pointer, but is it OK to
> > modify it or is a copy needed? The functions like cpp_warning_at()
> > are
> > exposed in the public header file, although at the moment, all call
> > sites are within libcpp and don't look like they would notice if the
> > argument was modified. I wasn't sure what is the expected interface
> > here.
>
> The diagnostic-reporting functions take non-const rich_location because
> the Fortran frontend doesn't set the locations on its diagnostics, but
> instead uses formatting codes %C and %L which write back locations into
> the rich_location when pp_format is called.  So there's some precedent
> for non-const rich_location, FWIW
>

OK cool, thanks for the explanation. I just noticed that the C++
frontend modifies the richloc too. (Sometimes it calls into libcpp
after lexing all the tokens, like for string concatenation, and then
it needs to set the location to the original one from when the token
was obtained.) I'll keep that in mind for the future, in the meantime
I have pushed it with the simple approach.

-Lewis

Re: [PATCH] diagnostics: libcpp: Improve locations for _Pragma lexing diagnostics [PR114423]

[Lewis Hyatt]

On Fri, Oct 18, 2024 at 11:25 AM David Malcolm  wrote:
> >if (!pfile->cb.diagnostic)
> >  abort ();
> > -  ret = pfile->cb.diagnostic (pfile, level, reason, richloc,
> > _(msgid), ap);
> > -
> > -  return ret;
> > +  if (pfile->diagnostic_override_loc && level != CPP_DL_NOTE)
> > +{
> > +  rich_location rc2{pfile->line_table, pfile-
> > >diagnostic_override_loc};
> > +  return pfile->cb.diagnostic (pfile, level, reason, &rc2,
> > _(msgid), ap);
> > +}
>
> This will effectively override the primary location in the
> rich_location, but by using a second rich_location instance it will
> also ignore any secondary locations and fix-it hints.
>
> This might will be what we want here, but did you consider
>   richloc.set_range (0, pfile->diagnostic_override_loc,
>  SHOW_RANGE_WITH_CARET);
> to reset the primary location?
>
> Otherwise, looks good to me.
>
> [...snip...]
>
> Thanks
> Dave
>

Thanks for taking a look! My thinking was, when libcpp produces tokens
from a _Pragma string, basically every location_t that it generates is
wrong and shouldn't be used. Because it doesn't actually set up the
line_map, it gets something random that's just sorta close to
reasonable. So I think it makes sense to discard fixits and secondary
locations too.

libcpp does use rich_location pretty sparingly, but I suppose the goal
is to use it more over time. We use one fixit hint for invalid
directive names currently, that one can't show up in a _Pragma though.
Right now we do define an encoding_rich_location subclass for escaping
unprintable bytes, which inherits rich_location and just adds a new
constructor to set the escape flag when it is created. You are
definitely right that this patch as of now loses that information.

Here's a source that uses an improperly normalized character:

_Pragma("ோ")

Currently we output:

t.cpp:1:1: warning: ‘\U0bc7\U0bbe’ is not in NFC [-Wnormalized=]
1 | _Pragma("")
  | ^~

With the patch we output:
t.cpp:1:1: warning: ‘\U0bc7\U0bbe’ is not in NFC [-Wnormalized=]
1 | _Pragma("ோ")
  | ^~~

So the main location range is improved (it underlines all of _Pragma
instead of most of it), but we have indeed lost the intended feature
that the incorrect bytes are escaped on the source line.

For this particular case I could improve it with a one line addition like

rc2.set_escape_on_output (richloc->escape_on_output_p ());

and that would actually handle all currently needed cases since we
don't use a lot of rich_locations in libcpp. It would just become
stale if some other option gets added to rich_location in the future
that we also should preserve. I think to fix it in a fully general
way, it would be necessary to add a new interface to class
rich_location. It already has a method to delete all the fixit hints,
it would also need a method to delete all the ranges. Then we could
make a copy of the richloc and just delete everything we don't want to
preserve. Do you have a preference one way or the other?

By the way, your suggestion was to directly modify richloc... these
functions do take the richloc by non-const pointer, but is it OK to
modify it or is a copy needed? The functions like cpp_warning_at() are
exposed in the public header file, although at the moment, all call
sites are within libcpp and don't look like they would notice if the
argument was modified. I wasn't sure what is the expected interface
here.

Thanks again...

-Lewis

[PATCH] diagnostics: libcpp: Improve locations for _Pragma lexing diagnostics [PR114423]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114423

The diagnostics we issue while lexing tokens from a _Pragma string have
always come out at invalid locations. I had tried a couple years ago to
fix this in a general way, but I think that ended up being too invasive a
change to fix a problem that's pretty minor in practice, and it never got
over the finish line. Here is a simple patch that improves the situation
and addresses the recently filed PR on that topic, hopefully this
incremental improvement is a better way to make some progress on this?

It just adds a way for libcpp to override the location for all diagnostics
temporarily, so that the diagnostics issued while lexing from a _Pragma
string are issued at a real location (the location of the _Pragma token)
and not a bogus one. That's a lot simpler than trying to arrange to
produce valid locations when lexing tokens from an internal
buffer. Bootstrap + regtest all languages on x86-64 Linux, tweaked a few
existing tests to adjust to the new locations. OK for trunk? Thanks!

-Lewis

-- >8 --

libcpp is not currently set up to be able to generate valid
locations for tokens lexed from a _Pragma string. Instead, after obtaining
the tokens, it sets their locations all to the location of the _Pragma
operator itself. This makes things like _Pragma("GCC diagnostic") work well
enough, but if any diagnostics are issued during lexing, prior to resetting
the token locations, those diagnostics get issued at the invalid
locations. Fix that up by adding a new field pfile->diagnostic_override_loc
that instructs libcpp to issue diagnostics at the alternate location.

libcpp/ChangeLog:

PR preprocessor/114423
* internal.h (struct cpp_reader): Add DIAGNOSTIC_OVERRIDE_LOC
field.
* directives.cc (destringize_and_run): Set the new field to the
location of the _Pragma operator.
* errors.cc (cpp_diagnostic_at): Support DIAGNOSTIC_OVERRIDE_LOC to
temporarily issue diagnostics at a different location.
(cpp_diagnostic_with_line): Likewise.

gcc/testsuite/ChangeLog:

PR preprocessor/114423
* c-c++-common/cpp/pragma-diagnostic-loc.c: New test.
* c-c++-common/cpp/diagnostic-pragma-1.c: Adjust expected output.
* g++.dg/pch/operator-1.C: Likewise.
---
 libcpp/directives.cc  |  7 +++
 libcpp/errors.cc  | 19 +--
 libcpp/internal.h |  4 
 .../c-c++-common/cpp/diagnostic-pragma-1.c|  9 -
 .../c-c++-common/cpp/pragma-diagnostic-loc.c  | 17 +
 gcc/testsuite/g++.dg/pch/operator-1.C |  8 +++-
 6 files changed, 52 insertions(+), 12 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pragma-diagnostic-loc.c

diff --git a/libcpp/directives.cc b/libcpp/directives.cc
index 9d235fa1b05..5706c28b835 100644
--- a/libcpp/directives.cc
+++ b/libcpp/directives.cc
@@ -2430,6 +2430,12 @@ destringize_and_run (cpp_reader *pfile, const cpp_string 
*in,
   pfile->buffer->file = pfile->buffer->prev->file;
   pfile->buffer->sysp = pfile->buffer->prev->sysp;
 
+  /* See comment below regarding the use of expansion_loc as the location
+ for all tokens; arrange here that diagnostics issued during lexing
+ get the same treatment.  */
+  const auto prev_loc_override = pfile->diagnostic_override_loc;
+  pfile->diagnostic_override_loc = expansion_loc;
+
   start_directive (pfile);
   _cpp_clean_line (pfile);
   save_directive = pfile->directive;
@@ -2497,6 +2503,7 @@ destringize_and_run (cpp_reader *pfile, const cpp_string 
*in,
  make that applicable to the real buffer too.  */
   pfile->buffer->prev->sysp = pfile->buffer->sysp;
   _cpp_pop_buffer (pfile);
+  pfile->diagnostic_override_loc = prev_loc_override;
 
   /* Reset the old macro state before ...  */
   XDELETE (pfile->context);
diff --git a/libcpp/errors.cc b/libcpp/errors.cc
index ad45f61913c..96fc165c12a 100644
--- a/libcpp/errors.cc
+++ b/libcpp/errors.cc
@@ -60,13 +60,14 @@ cpp_diagnostic_at (cpp_reader * pfile, enum 
cpp_diagnostic_level level,
   enum cpp_warning_reason reason, rich_location *richloc,
   const char *msgid, va_list *ap)
 {
-  bool ret;
-
   if (!pfile->cb.diagnostic)
 abort ();
-  ret = pfile->cb.diagnostic (pfile, level, reason, richloc, _(msgid), ap);
-
-  return ret;
+  if (pfile->diagnostic_override_loc && level != CPP_DL_NOTE)
+{
+  rich_location rc2{pfile->line_table, pfile->diagnostic_override_loc};
+  return pfile->cb.diagnostic (pfile, level, reason, &rc2, _(msgid), ap);
+}
+  return pfile->cb.diagnostic (pfile, level, reason, richloc, _(msgid), ap);
 }
 
 /* Print a diagnostic at the location of the previously lexed token.  */
@@ -201,8 +202,14 @@ cpp_diagnostic_with_line (cpp_reader * pfile, enum 
cpp_diagnostic_level level,
   
   if (!pfile->cb.diagnostic)
 abort ();
+  /* Don't override note locat

[PATCH] libcpp: Fix ICE lexing invalid raw string in a deferred pragma [PR117118]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=117118

This fixes an old regression from GCC 11. Is it OK for trunk and all
backports please? Bootstrap + regtested all languages on x86-64 Linux.
Thanks!

-Lewis

-- >8 --

The PR shows that we ICE after lexing an invalid unterminated raw string,
because lex_raw_string() pops the main buffer unexpectedly. Resolve by
handling this case the same way as for other directives.

libcpp/ChangeLog:
PR preprocessor/117118
* lex.cc (lex_raw_string): Treat an unterminated raw string the same
way for a deferred pragma as is done for other directives.

gcc/testsuite/ChangeLog:
PR preprocessor/117118
* c-c++-common/raw-string-directive-3.c: New test.
* c-c++-common/raw-string-directive-4.c: New test.
---
 libcpp/lex.cc   | 3 ++-
 gcc/testsuite/c-c++-common/raw-string-directive-3.c | 8 
 gcc/testsuite/c-c++-common/raw-string-directive-4.c | 8 
 3 files changed, 18 insertions(+), 1 deletion(-)
 create mode 100644 gcc/testsuite/c-c++-common/raw-string-directive-3.c
 create mode 100644 gcc/testsuite/c-c++-common/raw-string-directive-4.c

diff --git a/libcpp/lex.cc b/libcpp/lex.cc
index bb5cd394ab4..f9c5a070410 100644
--- a/libcpp/lex.cc
+++ b/libcpp/lex.cc
@@ -2655,7 +2655,8 @@ lex_raw_string (cpp_reader *pfile, cpp_token *token, 
const uchar *base)
{
  pos--;
  pfile->buffer->cur = pos;
- if ((pfile->state.in_directive || pfile->state.parsing_args)
+ if ((pfile->state.in_directive || pfile->state.parsing_args
+  || pfile->state.in_deferred_pragma)
  && pfile->buffer->next_line >= pfile->buffer->rlimit)
{
  cpp_error_with_line (pfile, CPP_DL_ERROR, token->src_loc, 0,
diff --git a/gcc/testsuite/c-c++-common/raw-string-directive-3.c 
b/gcc/testsuite/c-c++-common/raw-string-directive-3.c
new file mode 100644
index 000..fa4fa979fce
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/raw-string-directive-3.c
@@ -0,0 +1,8 @@
+/* { dg-options "-std=gnu99" { target c } } */
+/* { dg-options "-std=c++11" { target c++ } } */
+
+/* { dg-error "invalid new-line in raw string delimiter" "" { target *-*-* } 
.+4 } */
+/* { dg-error "unterminated raw string" "" { target *-*-* } .+3 } */
+/* { dg-error "stray 'R' in program" "" { target *-*-* } .+2 } */
+/* { dg-warning "expected a string" "" { target *-*-* } .+1 } */
+#pragma message R""
diff --git a/gcc/testsuite/c-c++-common/raw-string-directive-4.c 
b/gcc/testsuite/c-c++-common/raw-string-directive-4.c
new file mode 100644
index 000..935e3a1a991
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/raw-string-directive-4.c
@@ -0,0 +1,8 @@
+/* { dg-options "-std=gnu99" { target c } } */
+/* { dg-options "-std=c++11" { target c++ } } */
+
+/* { dg-error "invalid new-line in raw string delimiter" "" { target *-*-* } 
.+4 } */
+/* { dg-error "unterminated raw string" "" { target *-*-* } .+3 } */
+/* { dg-error "stray 'R' in program" "" { target *-*-* } .+2 } */
+/* { dg-warning "expected a string" "" { target *-*-* } .+1 } */
+_Pragma("message R\"\"")

Re: ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

On Fri, Oct 11, 2024 at 08:52:45PM +, Joseph Myers wrote:
> On Wed, 25 Sep 2024, Lewis Hyatt wrote:
> 
> > Hello-
> > 
> > May I please ping this one? Is there something maybe sub-optimal about
> > how I organized it? I can adjust or break it into two maybe if that's
> > helpful. Or else, if it's just that #pragma push_macro is not widely
> > used or cared about... I think it's still worth fixing clear gaps in
> > the support for UTF-8 input, and this is I think the last one, or at
> > least, the last obvious one. Thanks!
> > 
> > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> 
> This patch is OK.

Thank you very much for the review, I really appreciate it. FYI, the patch
required some conflict resolution since libcpp diagnostics were recently
tweaked, and I adjusted one testcase correspondingly. Here is what I have
pushed. Thanks!

-Lewis
Subject: [PATCH] libcpp: Support extended characters for #pragma 
{push,pop}_macro [PR109704]

The implementation of #pragma push_macro and #pragma pop_macro has to date
made use of an ad-hoc function, _cpp_lex_identifier(), which lexes an
identifier out of a string. When support was added for extended characters
in identifiers ($, UCNs, or UTF-8), that support was added only for the
"normal" way of lexing identifiers out of a cpp_buffer (_cpp_lex_direct) and
not for the ad-hoc way. Consequently, extended identifiers are not usable
with these pragmas.

The logic for lexing identifiers has become more complicated than it was
when _cpp_lex_identifier() was written -- it now handles things like \N{}
escapes in C++, for instance -- and it no longer seems practical to maintain
a redundant code path for lexing identifiers. Address the issue by changing
the implementation of #pragma {push,pop}_macro to lex identifiers in the
expected way, i.e. by pushing a cpp_buffer and lexing the identifier from
there.

The existing implementation has some quirks because of the ad-hoc parsing
logic. For example:

 #pragma push_macro("X ")
 ...
 #pragma pop_macro("X")

will not restore macro X (note the extra space in the first string). However:

 #pragma push_macro("X ")
 ...
 #pragma pop_macro("X ")

actually does sucessfully restore "X". This is because the key for looking
up the saved macro on the push stack is the original string passed, so the
string passed to pop_macro needs to match it exactly. It is not that easy to
reproduce this logic in the world of extended characters, given that for
example it should be valid to pass a UCN to push_macro, and the
corresponding UTF-8 to pop_macro. Given that this aspect of the existing
behavior seems unintentional and has no tests (and does not match other
implementations), I opted to make the new logic more straightforward. The
string passed needs to lex to one token, which must be a valid identifier,
or else no action is taken and no error is generated. Any diagnostics
encountered during lexing (e.g., due to a UTF-8 character not permitted to
appear in an identifier) are also suppressed.

It could be nice (for GCC 15) to also add a warning if a pop_macro does not
match a previous push_macro.

libcpp/ChangeLog:

PR preprocessor/109704
* include/cpplib.h (class cpp_auto_suppress_diagnostics): New class.
* errors.cc
(cpp_auto_suppress_diagnostics::cpp_auto_suppress_diagnostics): New
function.
(cpp_auto_suppress_diagnostics::~cpp_auto_suppress_diagnostics): New
function.
* charset.cc (noop_diagnostic_cb): Remove.
(cpp_interpret_string_ranges): Refactor diagnostic suppression logic
into new class cpp_auto_suppress_diagnostics.
(count_source_chars): Likewise.
* directives.cc (cpp_pop_definition): Add cpp_hashnode argument.
(lex_identifier_from_string): New static helper function.
(push_pop_macro_common): Refactor common logic from
do_pragma_push_macro and do_pragma_pop_macro; use
lex_identifier_from_string instead of _cpp_lex_identifier.
(do_pragma_push_macro): Reimplement using push_pop_macro_common.
(do_pragma_pop_macro): Likewise.
* internal.h (_cpp_lex_identifier): Remove.
* lex.cc (lex_identifier_intern): Remove.
(_cpp_lex_identifier): Remove.

gcc/testsuite/ChangeLog:

PR preprocessor/109704
* c-c++-common/cpp/pragma-push-pop-utf8.c: New test.
* g++.dg/pch/pushpop-2.C: New test.
* g++.dg/pch/pushpop-2.Hs: New test.
* gcc.dg/pch/pushpop-2.c: New test.
* gcc.dg/pch/pushpop-2.hs: New test.

diff --git a/gcc/testsuite/c-c++-common/cpp/pragma-push-pop-utf8.c 
b/gcc/testsuite/c-c++-common/cpp/pragma-push-pop-utf8.c
new file mode 100644
index 000..8634481d447
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pragma-push-pop-utf8.c
@@ -0,0 +1,203 @@
+/* {

Re: [PATCH] diagnostic: Save/restore diagnostic context history and push/pop state for PCH [PR116847]

[Lewis Hyatt]

On Fri, Sep 27, 2024 at 10:26 AM Jakub Jelinek  wrote:
>
> On Fri, Sep 27, 2024 at 09:54:13AM -0400, Lewis Hyatt wrote:
> > A couple comments that may be helpful...
> >
> > -This is also PR 64117 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64117)
> >
> > -I submitted a patch last year for that but did not get any response
> > (https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635648.html).
>
> Oops, sorry, wasn't aware of that.
> Note, I've already committed it.
>
> > I guess I never pinged it because I am still trying to ping two other
> > ones :). My patch did not switch to vec so it was not as nice as this
> > one. I wonder though, if some of the testcases I added could be
> > incorporated? In particular the testcase from my patch
>
> Maybe.
>
> > pragma-diagnostic-3.C I believe will still be failing after this one.
> > There is an issue with C++ because it processes the pragmas twice,
> > once in early mode and once in normal mode, that makes it do the wrong
> > thing for this case:
> >
> > t.h:
> > 
> >  #pragma GCC diagnostic push
> >  #pragma GCC diagnostic ignored...
> >  //no pop at end of the file
> >
> > t.c
> > 
> >  #include "t.h"
> >  #pragma GCC diagnostic pop
> >  //expect to be at the initial state here, but won't be if t.h is a PCH
> >
> > In my patch I had separated the PCH restore from a more general "state
> > restore" logic so that the C++ frontend can restore the state after
> > the first pass through the data.
>
> People shouldn't be doing this, without PCH or with it, and especially not
> with PCH, that is simply too ugly.
> That said, this was the reason why I have saved also the m_push_list
> vector and not just the history.  If that isn't enough and there is some
> other state on the libcpp side, I'd think we should go with a sorry and tell
> the user not to do this with PCH.  It becomes a nightmare already if e.g.
> the command line -Werror=something -Wno-error=somethingelse overrides
> differ between the PCH creation and PCH reading (my first version of
> the patch saved m_classify_diagnostic array but that broke tests relying
> on those differences).
>
> Jakub
>

Definitely agreed that people shouldn't be doing this :). I think I
felt like I should support it only because it was one of my patches
that caused C++ frontend to process the pragmas twice, but yeah I can
see how it's probably not worth the complexity.
One other note is that there's also been requests for the diagnostic
pragmas to make it through the LTO streaming process so that they can
be suppressed in the LTO frontend as well. That would also require
some more general support for saving the state, but it's another
topic.

-Lewis

Re: [PATCH] diagnostic: Save/restore diagnostic context history and push/pop state for PCH [PR116847]

[Lewis Hyatt]

On Fri, Sep 27, 2024 at 10:23 AM David Malcolm  wrote:
> > -I submitted a patch last year for that but did not get any response
> > (
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635648.html).
> > I guess I never pinged it because I am still trying to ping two other
> > ones :).
>
> Gahhh, I'm sorry about this.
>
> What are the other two patches?
>

Oh thanks, no worries... everyone has a lot to do :). I think, libcpp
patches in particular often don't get a lot of attention, but there is
a lot more going on. I'm happy to help where I can. Anyway the two I
was hoping to get in were:

https://gcc.gnu.org/pipermail/gcc-patches/2024-September/663853.html
https://gcc.gnu.org/pipermail/gcc-patches/2024-March/648297.html

In case there is time for them :).

-Lewis

Re: [PATCH] diagnostic: Save/restore diagnostic context history and push/pop state for PCH [PR116847]

[Lewis Hyatt]

On Fri, Sep 27, 2024 at 9:41 AM David Malcolm  wrote:
>
> On Thu, 2024-09-26 at 23:28 +0200, Jakub Jelinek wrote:
> > Hi!
> >
> > The following patch on top of the just posted cleanup patch
> > saves/restores the m_classification_history and m_push_list
> > vectors for PCH.  Without that as the testcase shows during parsing
> > of the templates we don't report ignored diagnostics, but after
> > loading
> > PCH header when instantiating those templates those warnings can be
> > emitted.  This doesn't show up on x86_64-linux build because
> > configure
> > injects there -fcf-protection -mshstk flags during library build (and
> > so
> > also during PCH header creation), but make check doesn't use those
> > flags
> > and so the PCH header is ignored.
> >
> > Bootstrapped on i686-linux so far, bootstrap/regtest on x86_64-linux
> > and
> > i686-linux still pending, ok for trunk if it passes it?
>
> Thanks, yes please
>
> Dave
>

A couple comments that may be helpful...

-This is also PR 64117 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64117)

-I submitted a patch last year for that but did not get any response
(https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635648.html).
I guess I never pinged it because I am still trying to ping two other
ones :). My patch did not switch to vec so it was not as nice as this
one. I wonder though, if some of the testcases I added could be
incorporated? In particular the testcase from my patch
pragma-diagnostic-3.C I believe will still be failing after this one.
There is an issue with C++ because it processes the pragmas twice,
once in early mode and once in normal mode, that makes it do the wrong
thing for this case:

t.h:

 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored...
 //no pop at end of the file

t.c

 #include "t.h"
 #pragma GCC diagnostic pop
 //expect to be at the initial state here, but won't be if t.h is a PCH

In my patch I had separated the PCH restore from a more general "state
restore" logic so that the C++ frontend can restore the state after
the first pass through the data.

-Lewis

ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

May I please ping this one? Is there something maybe sub-optimal about
how I organized it? I can adjust or break it into two maybe if that's
helpful. Or else, if it's just that #pragma push_macro is not widely
used or cared about... I think it's still worth fixing clear gaps in
the support for UTF-8 input, and this is I think the last one, or at
least, the last obvious one. Thanks!

https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

-Lewis

On Sun, Aug 25, 2024 at 11:45 AM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
>
> Monthly ping for this one please :). Thanks...
>
> -Lewis
>
> On Sat, Jul 27, 2024 at 3:09 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> >
> > Ping please? Jakub + Jason, hope you don't mind that I CCed you, I saw
> > you had your attention on extended character identifiers a bit now :).
> > Thanks!
> >
> > -Lewis
> >
> > On Fri, Jul 5, 2024 at 4:23 PM Lewis Hyatt  wrote:
> > >
> > > Hello-
> > >
> > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > >
> > > May I please ping this one again? It's the largest remaining gap in
> > > UTF-8 support for libcpp that I know of. Thanks!
> > >
> > > -Lewis
> > >
> > > On Tue, May 28, 2024 at 7:46 PM Lewis Hyatt  wrote:
> > > >
> > > > Hello-
> > > >
> > > > May I please ping this one (now for GCC 15)? Thanks!
> > > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > > >
> > > > -Lewis
> > > >
> > > > On Sat, Feb 10, 2024 at 9:02 AM Lewis Hyatt  wrote:
> > > > >
> > > > > Hello-
> > > > >
> > > > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > > > >
> > > > > May I please ping this one? Thanks!
> > > > >
> > > > > On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
> > > > > >
> > > > > > Hello-
> > > > > >
> > > > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
> > > > > >
> > > > > > The below patch fixes the issue noted in the PR that extended 
> > > > > > characters
> > > > > > cannot appear in the identifier passed to a #pragma push_macro or 
> > > > > > #pragma
> > > > > > pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it 
> > > > > > OK for
> > > > > > GCC 13 please?

Re: ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

Monthly ping for this one please :). Thanks...

-Lewis

On Sat, Jul 27, 2024 at 3:09 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
>
> Ping please? Jakub + Jason, hope you don't mind that I CCed you, I saw
> you had your attention on extended character identifiers a bit now :).
> Thanks!
>
> -Lewis
>
> On Fri, Jul 5, 2024 at 4:23 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> >
> > May I please ping this one again? It's the largest remaining gap in
> > UTF-8 support for libcpp that I know of. Thanks!
> >
> > -Lewis
> >
> > On Tue, May 28, 2024 at 7:46 PM Lewis Hyatt  wrote:
> > >
> > > Hello-
> > >
> > > May I please ping this one (now for GCC 15)? Thanks!
> > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > >
> > > -Lewis
> > >
> > > On Sat, Feb 10, 2024 at 9:02 AM Lewis Hyatt  wrote:
> > > >
> > > > Hello-
> > > >
> > > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > > >
> > > > May I please ping this one? Thanks!
> > > >
> > > > On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
> > > > >
> > > > > Hello-
> > > > >
> > > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
> > > > >
> > > > > The below patch fixes the issue noted in the PR that extended 
> > > > > characters
> > > > > cannot appear in the identifier passed to a #pragma push_macro or 
> > > > > #pragma
> > > > > pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it 
> > > > > OK for
> > > > > GCC 13 please?

ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

Ping please? Jakub + Jason, hope you don't mind that I CCed you, I saw
you had your attention on extended character identifiers a bit now :).
Thanks!

-Lewis

On Fri, Jul 5, 2024 at 4:23 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
>
> May I please ping this one again? It's the largest remaining gap in
> UTF-8 support for libcpp that I know of. Thanks!
>
> -Lewis
>
> On Tue, May 28, 2024 at 7:46 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > May I please ping this one (now for GCC 15)? Thanks!
> > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> >
> > -Lewis
> >
> > On Sat, Feb 10, 2024 at 9:02 AM Lewis Hyatt  wrote:
> > >
> > > Hello-
> > >
> > > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> > >
> > > May I please ping this one? Thanks!
> > >
> > > On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
> > > >
> > > > Hello-
> > > >
> > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
> > > >
> > > > The below patch fixes the issue noted in the PR that extended characters
> > > > cannot appear in the identifier passed to a #pragma push_macro or 
> > > > #pragma
> > > > pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it OK 
> > > > for
> > > > GCC 13 please?

ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

May I please ping this one again? It's the largest remaining gap in
UTF-8 support for libcpp that I know of. Thanks!

-Lewis

On Tue, May 28, 2024 at 7:46 PM Lewis Hyatt  wrote:
>
> Hello-
>
> May I please ping this one (now for GCC 15)? Thanks!
> https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
>
> -Lewis
>
> On Sat, Feb 10, 2024 at 9:02 AM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
> >
> > May I please ping this one? Thanks!
> >
> > On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
> > >
> > > Hello-
> > >
> > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
> > >
> > > The below patch fixes the issue noted in the PR that extended characters
> > > cannot appear in the identifier passed to a #pragma push_macro or #pragma
> > > pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it OK for
> > > GCC 13 please?

[PATCH] build: Fix "make install" for MinGW

[Lewis Hyatt]

Hello-

I noticed this while trying to test another patch on Windows (using the
MSYS2 environment). Tested that it fixes the issue for x86_64-w64-mingw32
and doesn't affect anything for x86_64-pc-linux-gnu. It looks like the same
fix for C was applied back in r11-702. OK? Thanks...

-Lewis

-- >8 --

Since r8-4925, the "make install" recipe generates a path which can start
with "//", causing problems for some Windows environments. Fix by removing
the redundant slash.
---
 gcc/cp/Make-lang.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcc/cp/Make-lang.in b/gcc/cp/Make-lang.in
index 026cf8d7088..e792ea4ddf3 100644
--- a/gcc/cp/Make-lang.in
+++ b/gcc/cp/Make-lang.in
@@ -344,7 +344,7 @@ c++.install-plugin: installdirs
 # Install import library.
 ifeq ($(plugin_implib),yes)
$(mkinstalldirs) $(DESTDIR)$(plugin_resourcesdir)
-   $(INSTALL_DATA) cc1plus$(exeext).a 
$(DESTDIR)/$(plugin_resourcesdir)/cc1plus$(exeext).a
+   $(INSTALL_DATA) cc1plus$(exeext).a 
$(DESTDIR)$(plugin_resourcesdir)/cc1plus$(exeext).a
 endif
 
 c++.uninstall:

[PATCH] preprocessor: Create the parser before handling command-line includes [PR115312]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115312

This fixes a 14.1 regression with PCH for MinGW and other platforms that don't
use stdc-predef.h. Bootstrap + regtest all languages on x86-64 Linux;
bootstrap + regtest c,c++ on x86_64-w64-mingw32. Is it OK for 14 branch and
master please? Thanks!

-Lewis

-- >8 --

Since r14-2893, we create a parser object in preprocess-only mode for the
purpose of parsing #pragma while preprocessing. The parser object was
formerly created after calling c_finish_options(), which leads to problems
on platforms that don't use stdc-predef.h (such as MinGW, as reported in
the PR). On such platforms, the call to c_finish_options() will process
the first command-line-specified include file. If that includes a PCH, then
c-ppoutput.cc will encounter a state it did not anticipate. Fix it by
creating the parser prior to calling c_finish_options().

gcc/c-family/ChangeLog:

PR pch/115312
* c-opts.cc (c_common_init): Call c_init_preprocess() before
c_finish_options() so that a parser is available to process any
includes specified on the command line.

gcc/testsuite/ChangeLog:

PR pch/115312
* g++.dg/pch/pr115312.C: New test.
* g++.dg/pch/pr115312.Hs: New test.
---
 gcc/c-family/c-opts.cc   | 2 +-
 gcc/testsuite/g++.dg/pch/pr115312.C  | 2 ++
 gcc/testsuite/g++.dg/pch/pr115312.Hs | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 gcc/testsuite/g++.dg/pch/pr115312.C
 create mode 100644 gcc/testsuite/g++.dg/pch/pr115312.Hs

diff --git a/gcc/c-family/c-opts.cc b/gcc/c-family/c-opts.cc
index 33114f13c8d..b7789b707e6 100644
--- a/gcc/c-family/c-opts.cc
+++ b/gcc/c-family/c-opts.cc
@@ -1296,8 +1296,8 @@ c_common_init (void)
 
   if (flag_preprocess_only)
 {
-  c_finish_options ();
   c_init_preprocess ();
+  c_finish_options ();
   preprocess_file (parse_in);
   return false;
 }
diff --git a/gcc/testsuite/g++.dg/pch/pr115312.C 
b/gcc/testsuite/g++.dg/pch/pr115312.C
new file mode 100644
index 000..9074ad4a5ad
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/pr115312.C
@@ -0,0 +1,2 @@
+/* { dg-additional-options "-include pr115312.H -save-temps" } */
+#error "suppress PCH assembly comparison, which does not work with 
-save-temps" /* { dg-error "." } */
diff --git a/gcc/testsuite/g++.dg/pch/pr115312.Hs 
b/gcc/testsuite/g++.dg/pch/pr115312.Hs
new file mode 100644
index 000..6e7c6bcac2f
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/pr115312.Hs
@@ -0,0 +1 @@
+// This space intentionally left blank.

Re: ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

May I please ping this one (now for GCC 15)? Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

-Lewis

On Sat, Feb 10, 2024 at 9:02 AM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html
>
> May I please ping this one? Thanks!
>
> On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
> >
> > The below patch fixes the issue noted in the PR that extended characters
> > cannot appear in the identifier passed to a #pragma push_macro or #pragma
> > pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it OK for
> > GCC 13 please?
> >
> > I know we just entered stage 4, however I feel this is kinda like an old
> > regression, given that the issue was not apparent until support for UCNs and
> > UTF-8 in identifiers got added. FWIW, it would be nice if it makes it into
> > GCC 13, because AFAIK all other UTF-8-related bugs are fixed in this
> > release. (The other major one was for extended characters in a user-defined
> > literal, that was fixed by r14-2629).
> >
> > Speaking of just entering stage 4. I do have 4 really short patches sent
> > over the past several months that never got any response. Is there any
> > chance someone may have a few minutes to look at them please? They are
> > really just like 1-3 line fixes for PRs.
> >
> > libcpp (pinged once recently):
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-December/640386.html
> >
> > diagnostics (pinged for 3rd time last week):
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
>
> > -- >8 --
> >
> > The implementation of #pragma push_macro and #pragma pop_macro has to date
> > made use of an ad-hoc function, _cpp_lex_identifier(), which lexes an
> > identifier out of a string. When support was added for extended characters
> > in identifiers ($, UCNs, or UTF-8), that support was added only for the
> > "normal" way of lexing identifiers out of a cpp_buffer (_cpp_lex_direct) and
> > not for the ad-hoc way. Consequently, extended identifiers are not usable
> > with these pragmas.
> >
> > The logic for lexing identifiers has become more complicated than it was
> > when _cpp_lex_identifier() was written -- it now handles things like \N{}
> > escapes in C++, for instance -- and it no longer seems practical to maintain
> > a redundant code path for lexing identifiers. Address the issue by changing
> > the implementation of #pragma {push,pop}_macro to lex identifiers in the
> > expected way, i.e. by pushing a cpp_buffer and lexing the identifier from
> > there.
> >
> > The existing implementation has some quirks because of the ad-hoc parsing
> > logic. For example:
> >
> >  #pragma push_macro("X ")
> >  ...
> >  #pragma pop_macro("X")
> >
> > will not restore macro X (note the extra space in the first string). 
> > However:
> >
> >  #pragma push_macro("X ")
> >  ...
> >  #pragma pop_macro("X ")
> >
> > actually does sucessfully restore "X". This is because the key for looking
> > up the saved macro on the push stack is the original string passed, so the
> > string passed to pop_macro needs to match it exactly. It is not that easy to
> > reproduce this logic in the world of extended characters, given that for
> > example it should be valid to pass a UCN to push_macro, and the
> > corresponding UTF-8 to pop_macro. Given that this aspect of the existing
> > behavior seems unintentional and has no tests (and does not match other
> > implementations), I opted to make the new logic more straightforward. The
> > string passed needs to lex to one token, which must be a valid identifier,
> > or else no action is taken and no error is generated. Any diagnostics
> > encountered during lexing (e.g., due to a UTF-8 character not permitted to
> > appear in an identifier) are also suppressed.
> >
> > It could be nice (for GCC 15) to also add a warning if a pop_macro does not
> > match a previous push_macro.
> >
> > libcpp/ChangeLog:
> >
> > PR preprocessor/109704
> > * include/cpplib.h (class cpp_auto_suppress_diagnostics): New class.
> > * errors.cc
> > (cpp_auto_suppress_diagnostics::cpp_auto_suppress_diagnostics): New
> > function.
> > (cpp_auto_suppress_diagnostics::~cpp_auto_supp

[PATCH] libcpp: Fix _Pragma("GCC system_header") [PR114436]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=114436

This is a small fix for the issue mentioned in the PR that _Pragma("GCC
system_header") does not work completely. I believe it was always the case
since _Pragma() support was first added. bootstrap + regtested all languages
on x86-64 Linux. Is it OK now or in stage 1 please? Thanks!

-Lewis

-- >8 --

_Pragma("GCC system_header") currently takes effect only partially. It does
succeed in updating the line_map, so that checks like in_system_header_at()
return correctly, but it does not update pfile->buffer->sysp.  One result is
that a subsequent #include does not set up the system header state properly
for the newly included file, as pointed out in the PR. Fix by propagating
the new system header state back to the buffer after processing the pragma.

libcpp/ChangeLog:

PR preprocessor/114436
* directives.cc (destringize_and_run): If the _Pragma changed the
buffer system header state (e.g. because it was _Pragma("GCC
system_header"), propagate that change back to the actual buffer too.

gcc/testsuite/ChangeLog:

PR preprocessor/114436
* c-c++-common/cpp/pragma-system-header-1.h: New test.
* c-c++-common/cpp/pragma-system-header-2.h: New test.
* c-c++-common/cpp/pragma-system-header.c: New test.
---
 libcpp/directives.cc  | 11 ---
 .../c-c++-common/cpp/pragma-system-header-1.h |  1 +
 .../c-c++-common/cpp/pragma-system-header-2.h |  5 +
 gcc/testsuite/c-c++-common/cpp/pragma-system-header.c |  3 +++
 4 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pragma-system-header-1.h
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pragma-system-header-2.h
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pragma-system-header.c

diff --git a/libcpp/directives.cc b/libcpp/directives.cc
index 479f8c716e8..bbaf9db6af0 100644
--- a/libcpp/directives.cc
+++ b/libcpp/directives.cc
@@ -1919,9 +1919,11 @@ destringize_and_run (cpp_reader *pfile, const cpp_string 
*in,
  until we've read all of the tokens that we want.  */
   cpp_push_buffer (pfile, (const uchar *) result, dest - result,
   /* from_stage3 */ true);
-  /* ??? Antique Disgusting Hack.  What does this do?  */
-  if (pfile->buffer->prev)
-pfile->buffer->file = pfile->buffer->prev->file;
+
+  /* This is needed for _Pragma("once") and _Pragma("GCC system_header") to 
work
+ properly.  */
+  pfile->buffer->file = pfile->buffer->prev->file;
+  pfile->buffer->sysp = pfile->buffer->prev->sysp;
 
   start_directive (pfile);
   _cpp_clean_line (pfile);
@@ -1986,6 +1988,9 @@ destringize_and_run (cpp_reader *pfile, const cpp_string 
*in,
 
   /* Finish inlining run_directive.  */
   pfile->buffer->file = NULL;
+  /* If the system header state changed due to #pragma GCC system_header, then
+ make that applicable to the real buffer too.  */
+  pfile->buffer->prev->sysp = pfile->buffer->sysp;
   _cpp_pop_buffer (pfile);
 
   /* Reset the old macro state before ...  */
diff --git a/gcc/testsuite/c-c++-common/cpp/pragma-system-header-1.h 
b/gcc/testsuite/c-c++-common/cpp/pragma-system-header-1.h
new file mode 100644
index 000..bd9ff0cb138
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pragma-system-header-1.h
@@ -0,0 +1 @@
+#pragma GCC warning "this warning should not be output (1)"
diff --git a/gcc/testsuite/c-c++-common/cpp/pragma-system-header-2.h 
b/gcc/testsuite/c-c++-common/cpp/pragma-system-header-2.h
new file mode 100644
index 000..a62d9e2685a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pragma-system-header-2.h
@@ -0,0 +1,5 @@
+_Pragma("GCC system_header")
+#include "pragma-system-header-1.h"
+#pragma GCC warning "this warning should not be output (2)"
+_Pragma("unknown")
+#include "pragma-system-header-1.h"
diff --git a/gcc/testsuite/c-c++-common/cpp/pragma-system-header.c 
b/gcc/testsuite/c-c++-common/cpp/pragma-system-header.c
new file mode 100644
index 000..fdea12009e1
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pragma-system-header.c
@@ -0,0 +1,3 @@
+#include "pragma-system-header-2.h" /* { dg-bogus "this warning should not be 
output" } */
+/* { dg-do preprocess } */
+/* PR preprocessor/114436 */

ping: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

Thanks!

On Fri, Feb 16, 2024 at 7:02 PM Lewis Hyatt  wrote:
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
>
> On Thu, Jan 25, 2024 at 4:57 PM Lewis Hyatt  wrote:
> >
> > May I please ask again about this one? It's just a couple lines, and I
> > think it fixes an important gap in the logic for #pragma GCC
> > diagnostic. The PR was not reported by me so I think at least one
> > other person does care about it :). Thanks!
> >
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> >
> > -Lewis
> >
> > On Mon, Jan 8, 2024 at 6:53 PM Lewis Hyatt  wrote:
> > >
> > > Can I please ping this one again? It's 3 lines or so to fix the PR. 
> > > Thanks!
> > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> > >
> > > On Tue, Dec 19, 2023 at 6:20 PM Lewis Hyatt  wrote:
> > > >
> > > > Hello-
> > > >
> > > > May I please ping this one? Thanks...
> > > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> > > >
> > > > -Lewis
> > > >
> > > > On Wed, Nov 29, 2023 at 7:05 PM Lewis Hyatt  wrote:
> > > > >
> > > > > On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> > > > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> > > > > >
> > > > > > This patch fixes the behavior of `#pragma GCC diagnostic pop' for 
> > > > > > permissive
> > > > > > error diagnostics such as -Wnarrowing (in C++11). Those currently 
> > > > > > do not
> > > > > > return to the correct state after the last pop; they become 
> > > > > > effectively
> > > > > > simple warnings instead. Bootstrap + regtest all languages on 
> > > > > > x86-64, does
> > > > > > it look OK please? Thanks!
> > > > >
> > > > > Hello-
> > > > >
> > > > > May I please ping this bug fix?
> > > > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html
> > > > >
> > > > > Please note, it requires a trivial rebase on top of recent changes to
> > > > > the class diagnostic_context public interface. I attached the rebased 
> > > > > patch
> > > > > here as well. Thanks!
> > > > >
> > > > > -Lewis

Re: ping: [PATCH] libcpp: Fix __has_include_next ICE in the last directory of the path [PR80755]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html

There was a request on the PR
(https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80755#c5) for me to ping
this again, so I am complying :). Might anyone have a minute to take a
look please? Thanks...


-Lewis


On Thu, Jan 11, 2024 at 7:34 AM Lewis Hyatt  wrote:
>
> Can I please ping this one? Thanks...
> https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html
>
> -Lewis
>
> On Thu, Dec 21, 2023 at 7:37 AM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80755
> >
> > Here is a short fix for the ICE in libcpp noted in the PR. Bootstrap +
> > regtest all languages on x86-64 Linux. Is it OK please? Thanks!
> >
> > -Lewis
> >
> > -- >8 --
> >
> > In libcpp/files.cc, the function _cpp_has_header(), which implements
> > __has_include and __has_include_next, does not check for a NULL return value
> > from search_path_head(), leading to an ICE tripping an assert when
> > _cpp_find_file() tries to use it. Fix it by checking for that case and
> > silently returning false instead.
> >
> > As suggested by the PR author, it is easiest to make a testcase by using
> > the -idirafter option. To enable that, also modify the dg-additional-options
> > testsuite procedure to make the global $srcdir available, since -idirafter
> > requires the full path.
> >
> > libcpp/ChangeLog:
> >
> > PR preprocessor/80755
> > * files.cc (search_path_head): Add SUPPRESS_DIAGNOSTIC argument
> > defaulting to false.
> > (_cpp_has_header): Silently return false if the search path has been
> > exhausted, rather than issuing a diagnostic and then hitting an
> > assert.
> >
> > gcc/testsuite/ChangeLog:
> >
> > * lib/gcc-defs.exp (dg-additional-options): Make $srcdir usable in a
> > dg-additional-options directive.
> > * c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h: New 
> > test.
> > * c-c++-common/cpp/has-include-next-2.c: New test.
> > ---
> >  libcpp/files.cc  | 12 
> >  .../cpp/has-include-next-2-dir/has-include-next-2.h  |  3 +++
> >  gcc/testsuite/c-c++-common/cpp/has-include-next-2.c  |  4 
> >  gcc/testsuite/lib/gcc-defs.exp   |  1 +
> >  4 files changed, 16 insertions(+), 4 deletions(-)
> >  create mode 100644 
> > gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
> >  create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-next-2.c
> >
> > diff --git a/libcpp/files.cc b/libcpp/files.cc
> > index 27301d79fa4..aaab4b13a6a 100644
> > --- a/libcpp/files.cc
> > +++ b/libcpp/files.cc
> > @@ -181,7 +181,8 @@ static bool read_file_guts (cpp_reader *pfile, 
> > _cpp_file *file,
> >  static bool read_file (cpp_reader *pfile, _cpp_file *file,
> >location_t loc);
> >  static struct cpp_dir *search_path_head (cpp_reader *, const char *fname,
> > -int angle_brackets, enum include_type);
> > +int angle_brackets, enum 
> > include_type,
> > +bool suppress_diagnostic = false);
> >  static const char *dir_name_of_file (_cpp_file *file);
> >  static void open_file_failed (cpp_reader *pfile, _cpp_file *file, int,
> >   location_t);
> > @@ -1041,7 +1042,7 @@ _cpp_mark_file_once_only (cpp_reader *pfile, 
> > _cpp_file *file)
> > nothing left in the path, returns NULL.  */
> >  static struct cpp_dir *
> >  search_path_head (cpp_reader *pfile, const char *fname, int angle_brackets,
> > - enum include_type type)
> > + enum include_type type, bool suppress_diagnostic)
> >  {
> >cpp_dir *dir;
> >_cpp_file *file;
> > @@ -1070,7 +1071,7 @@ search_path_head (cpp_reader *pfile, const char 
> > *fname, int angle_brackets,
> >  return make_cpp_dir (pfile, dir_name_of_file (file),
> >  pfile->buffer ? pfile->buffer->sysp : 0);
> >
> > -  if (dir == NULL)
> > +  if (dir == NULL && !suppress_diagnostic)
> >  cpp_error (pfile, CPP_DL_ERROR,
> >"no include path in which to search for %s", fname);
> >
> > @@ -2164,7 +2165,10 @@ bool
> >  _cpp_has_header (cpp_reader *pfile, const char *fname, int angle_brackets,
> >  

Re: [PATCH] libcpp: Improve location for macro names [PR66290]

[Lewis Hyatt]

On Thu, Feb 22, 2024 at 3:56 AM Richard Biener
 wrote:
>
> On Tue, Feb 20, 2024 at 3:33 PM Lewis Hyatt  wrote:
> >
> > On Mon, Feb 19, 2024 at 11:36 PM Alexandre Oliva  wrote:
> > >
> > > This backport for gcc-13 is the first of two required for the
> > > g++.dg/pch/line-map-3.C test to stop hitting a variant of the known
> > > problem mentioned in that testcase: on riscv64-elf and riscv32-elf,
> > > after restoring the PCH, the location of the macros is mentioned as if
> > > they were on line 3 rather than 2, so even the existing xfails fail.  I
> > > think this might be too much to backport, and I'm ready to use an xfail
> > > instead, but since this would bring more predictability, I thought I'd
> > > ask whether you'd find this backport acceptable.
> > >
> > > Regstrapped on x86_64-linux-gnu, along with other backports, and tested
> > > manually on riscv64-elf.  Ok to install?
> >
> > Sorry that test is causing a problem, I hadn't realized at first that
> > the wrong output was target-dependent. I feel like simply deleting
> > this test g++.dg/pch/line-map-3.C from GCC 13 branch should be fine
> > too, as a safer alternative, if release managers prefer?
>
> Yes please.
>
> Richard.

Committed that removal as r13-8353.

-Lewis

Re: [PATCH] libcpp: Improve location for macro names [PR66290]

[Lewis Hyatt]

On Mon, Feb 19, 2024 at 11:36 PM Alexandre Oliva  wrote:
>
> This backport for gcc-13 is the first of two required for the
> g++.dg/pch/line-map-3.C test to stop hitting a variant of the known
> problem mentioned in that testcase: on riscv64-elf and riscv32-elf,
> after restoring the PCH, the location of the macros is mentioned as if
> they were on line 3 rather than 2, so even the existing xfails fail.  I
> think this might be too much to backport, and I'm ready to use an xfail
> instead, but since this would bring more predictability, I thought I'd
> ask whether you'd find this backport acceptable.
>
> Regstrapped on x86_64-linux-gnu, along with other backports, and tested
> manually on riscv64-elf.  Ok to install?

Sorry that test is causing a problem, I hadn't realized at first that
the wrong output was target-dependent. I feel like simply deleting
this test g++.dg/pch/line-map-3.C from GCC 13 branch should be fine
too, as a safer alternative, if release managers prefer? It doesn't
really need to be on the branch, it's only purpose is to remind me to
fix the underlying issue for GCC 15...

-Lewis

ping: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

CCing some global reviewers as well, in case anyone has a minute to
take a look please? Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

On Thu, Jan 25, 2024 at 4:57 PM Lewis Hyatt  wrote:
>
> May I please ask again about this one? It's just a couple lines, and I
> think it fixes an important gap in the logic for #pragma GCC
> diagnostic. The PR was not reported by me so I think at least one
> other person does care about it :). Thanks!
>
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
>
> -Lewis
>
> On Mon, Jan 8, 2024 at 6:53 PM Lewis Hyatt  wrote:
> >
> > Can I please ping this one again? It's 3 lines or so to fix the PR. Thanks!
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> >
> > On Tue, Dec 19, 2023 at 6:20 PM Lewis Hyatt  wrote:
> > >
> > > Hello-
> > >
> > > May I please ping this one? Thanks...
> > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> > >
> > > -Lewis
> > >
> > > On Wed, Nov 29, 2023 at 7:05 PM Lewis Hyatt  wrote:
> > > >
> > > > On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> > > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> > > > >
> > > > > This patch fixes the behavior of `#pragma GCC diagnostic pop' for 
> > > > > permissive
> > > > > error diagnostics such as -Wnarrowing (in C++11). Those currently do 
> > > > > not
> > > > > return to the correct state after the last pop; they become 
> > > > > effectively
> > > > > simple warnings instead. Bootstrap + regtest all languages on x86-64, 
> > > > > does
> > > > > it look OK please? Thanks!
> > > >
> > > > Hello-
> > > >
> > > > May I please ping this bug fix?
> > > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html
> > > >
> > > > Please note, it requires a trivial rebase on top of recent changes to
> > > > the class diagnostic_context public interface. I attached the rebased 
> > > > patch
> > > > here as well. Thanks!
> > > >
> > > > -Lewis

ping: [PATCH] libcpp: Support extended characters for #pragma {push,pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/pipermail/gcc-patches/2024-January/642926.html

May I please ping this one? Thanks!

On Sat, Jan 13, 2024 at 5:12 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704
>
> The below patch fixes the issue noted in the PR that extended characters
> cannot appear in the identifier passed to a #pragma push_macro or #pragma
> pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it OK for
> GCC 13 please?
>
> I know we just entered stage 4, however I feel this is kinda like an old
> regression, given that the issue was not apparent until support for UCNs and
> UTF-8 in identifiers got added. FWIW, it would be nice if it makes it into
> GCC 13, because AFAIK all other UTF-8-related bugs are fixed in this
> release. (The other major one was for extended characters in a user-defined
> literal, that was fixed by r14-2629).
>
> Speaking of just entering stage 4. I do have 4 really short patches sent
> over the past several months that never got any response. Is there any
> chance someone may have a few minutes to look at them please? They are
> really just like 1-3 line fixes for PRs.
>
> libcpp (pinged once recently):
> https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html
> https://gcc.gnu.org/pipermail/gcc-patches/2023-December/640386.html
>
> diagnostics (pinged for 3rd time last week):
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

> -- >8 --
>
> The implementation of #pragma push_macro and #pragma pop_macro has to date
> made use of an ad-hoc function, _cpp_lex_identifier(), which lexes an
> identifier out of a string. When support was added for extended characters
> in identifiers ($, UCNs, or UTF-8), that support was added only for the
> "normal" way of lexing identifiers out of a cpp_buffer (_cpp_lex_direct) and
> not for the ad-hoc way. Consequently, extended identifiers are not usable
> with these pragmas.
>
> The logic for lexing identifiers has become more complicated than it was
> when _cpp_lex_identifier() was written -- it now handles things like \N{}
> escapes in C++, for instance -- and it no longer seems practical to maintain
> a redundant code path for lexing identifiers. Address the issue by changing
> the implementation of #pragma {push,pop}_macro to lex identifiers in the
> expected way, i.e. by pushing a cpp_buffer and lexing the identifier from
> there.
>
> The existing implementation has some quirks because of the ad-hoc parsing
> logic. For example:
>
>  #pragma push_macro("X ")
>  ...
>  #pragma pop_macro("X")
>
> will not restore macro X (note the extra space in the first string). However:
>
>  #pragma push_macro("X ")
>  ...
>  #pragma pop_macro("X ")
>
> actually does sucessfully restore "X". This is because the key for looking
> up the saved macro on the push stack is the original string passed, so the
> string passed to pop_macro needs to match it exactly. It is not that easy to
> reproduce this logic in the world of extended characters, given that for
> example it should be valid to pass a UCN to push_macro, and the
> corresponding UTF-8 to pop_macro. Given that this aspect of the existing
> behavior seems unintentional and has no tests (and does not match other
> implementations), I opted to make the new logic more straightforward. The
> string passed needs to lex to one token, which must be a valid identifier,
> or else no action is taken and no error is generated. Any diagnostics
> encountered during lexing (e.g., due to a UTF-8 character not permitted to
> appear in an identifier) are also suppressed.
>
> It could be nice (for GCC 15) to also add a warning if a pop_macro does not
> match a previous push_macro.
>
> libcpp/ChangeLog:
>
> PR preprocessor/109704
> * include/cpplib.h (class cpp_auto_suppress_diagnostics): New class.
> * errors.cc
> (cpp_auto_suppress_diagnostics::cpp_auto_suppress_diagnostics): New
> function.
> (cpp_auto_suppress_diagnostics::~cpp_auto_suppress_diagnostics): New
> function.
> * charset.cc (noop_diagnostic_cb): Remove.
> (cpp_interpret_string_ranges): Refactor diagnostic suppression logic
> into new class cpp_auto_suppress_diagnostics.
> (count_source_chars): Likewise.
> * directives.cc (cpp_pop_definition): Add cpp_hashnode argument.
> (lex_identifier_from_string): New static helper function.
> (push_pop_macro_common): Refactor common logic from
> do_pragma_push_macro and do_pragma_pop_macro; use
> lex_identifier_from_string instead of _cpp_lex_identifier.
> (d

Re: [PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

On Thu, Feb 1, 2024 at 7:24 AM Rainer Orth  
wrote:
>
> Hi Lewis,
>
> > On Fri, Jan 26, 2024 at 04:16:54PM -0500, Jason Merrill wrote:
> >> On 12/5/23 20:52, Lewis Hyatt wrote:
> >> > Hello-
> >> >
> >> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608
> >> >
> >> > There are two related issues here really, a regression since GCC 11 
> >> > where we
> >> > can ICE after restoring a PCH, and a deeper issue with bogus locations
> >> > assigned to macros that were defined prior to restoring a PCH.  This 
> >> > patch
> >> > fixes the ICE regression with a simple change, and I think it's 
> >> > appropriate
> >> > for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, 
> >> > but
> >> > not generally causing an ICE, and mostly affecting only the output of
> >> > -Wunused-macros) are not as problematic, and will be harder to fix. I 
> >> > could
> >> > take a stab at that for GCC 15. In the meantime the patch adds XFAILed
> >> > tests for the wrong locations (as well as passing tests for the 
> >> > regression
> >> > fix). Does it look OK please? Bootstrap + regtest all languages on x86-64
> >> > Linux. Thanks!
> >>
> >> OK for trunk and branches, thanks!
> >>
> >
> > Thanks for the review! That is all taken care of. I have one more request if
> > you don't mind please... There have been some further comments on the PR
> > indicating that the new xfailed testcase I added is failing in an unexpected
> > way on at least one architecture. To recap, the idea here was that
> >
> > 1) libcpp needs new logic to be able to output correct locations for this
> > case. That will be some new code that is suitable for stage 1, not now.
> >
> > 2) In the meantime, we fixed things up enough to avoid an ICE that showed up
> > in GCC 11, and added an xfailed testcase to remind about #1.
> >
> > The problem is that, the reason that libcpp outputs the wrong locations, is
> > that it has always used a location from the old line_map instance to index
> > into the new line_map instance, and so the exact details of the wrong
> > locations it outputs depend on the state of those two line maps, which may
> > differ depending on system includes and things like that. So I was hoping to
> > make one further one-line change to libcpp, not yet to output correct
> > locations, but at least to output one which is the same always and doesn't
> > depend on random things. This would assign all restored macros to a
> > consistent location, one line following the #include that triggered the PCH
> > process. I think this probably shouldn't be backported but it would be nice
> > to get into GCC 14, while nothing critical, at least it would avoid the new
> > test failure that's being reported. But more generally, I think using a
> > location from a totally different line map is dangerous and could have worse
> > consequences that haven't been seen yet. Does it look OK please? Thanks!
>
> FWIW, I've tested this (the initial) version of this patch on
> sparc-sun-solaris2.11 (PASSes as before) and i386-pc-solaris2.11 (PASSes
> now unlike before).
>
> Thanks.
> Rainer

Thanks a lot! And sorry for that issue. I will push the updated
version of that patch shortly.

Re: [PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

On Wed, Jan 31, 2024 at 03:18:01PM -0500, Jason Merrill wrote:
> On 1/30/24 21:49, Lewis Hyatt wrote:
> > On Fri, Jan 26, 2024 at 04:16:54PM -0500, Jason Merrill wrote:
> > > On 12/5/23 20:52, Lewis Hyatt wrote:
> > > > Hello-
> > > > 
> > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608
> > > > 
> > > > There are two related issues here really, a regression since GCC 11 
> > > > where we
> > > > can ICE after restoring a PCH, and a deeper issue with bogus locations
> > > > assigned to macros that were defined prior to restoring a PCH.  This 
> > > > patch
> > > > fixes the ICE regression with a simple change, and I think it's 
> > > > appropriate
> > > > for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, 
> > > > but
> > > > not generally causing an ICE, and mostly affecting only the output of
> > > > -Wunused-macros) are not as problematic, and will be harder to fix. I 
> > > > could
> > > > take a stab at that for GCC 15. In the meantime the patch adds XFAILed
> > > > tests for the wrong locations (as well as passing tests for the 
> > > > regression
> > > > fix). Does it look OK please? Bootstrap + regtest all languages on 
> > > > x86-64
> > > > Linux. Thanks!
> > > 
> > > OK for trunk and branches, thanks!
> > > 
> > 
> > Thanks for the review! That is all taken care of. I have one more request if
> > you don't mind please... There have been some further comments on the PR
> > indicating that the new xfailed testcase I added is failing in an unexpected
> > way on at least one architecture. To recap, the idea here was that
> > 
> > 1) libcpp needs new logic to be able to output correct locations for this
> > case. That will be some new code that is suitable for stage 1, not now.
> > 
> > 2) In the meantime, we fixed things up enough to avoid an ICE that showed up
> > in GCC 11, and added an xfailed testcase to remind about #1.
> > 
> > The problem is that, the reason that libcpp outputs the wrong locations, is
> > that it has always used a location from the old line_map instance to index
> > into the new line_map instance, and so the exact details of the wrong
> > locations it outputs depend on the state of those two line maps, which may
> > differ depending on system includes and things like that. So I was hoping to
> > make one further one-line change to libcpp, not yet to output correct
> > locations, but at least to output one which is the same always and doesn't
> > depend on random things. This would assign all restored macros to a
> > consistent location, one line following the #include that triggered the PCH
> > process. I think this probably shouldn't be backported but it would be nice
> > to get into GCC 14, while nothing critical, at least it would avoid the new
> > test failure that's being reported. But more generally, I think using a
> > location from a totally different line map is dangerous and could have worse
> > consequences that haven't been seen yet. Does it look OK please? Thanks!
> 
> Can we use the line of the #include, as the test expects, rather than the
> following line?

Thanks, yes, that will work too, it just needs a few changes to
c-family/c-pch.cc to set the location there and then increment it
after. Patch which does that is attached. (This is a new one based on
master, not incremental to the prior patch.) The testcase does not require
any changes this way, and bootstrap + regtest looks good.

-Lewis
[PATCH] c-family: Stabilize the location for macros restored after PCH load 
[PR105608]

libcpp currently lacks the infrastructure to assign correct locations to
macros that were defined prior to loading a PCH and then restored
afterwards. While I plan to address that fully for GCC 15, this patch
improves things by using at least a valid location, even if it's not the
best one. Without this change, libcpp uses pfile->directive_line as the
location for the restored macros, but this location_t applies to the old
line map, not the one that was just restored from the PCH, so the resulting
location is unpredictable and depends on what was stored in the line maps
before. With this change, all restored macros get assigned locations at the
line of the #include that triggered the PCH restore. A future patch will
store the actual file name and line number of each definition and then
synthesize locations in the new line map pointing to the right place.

gcc/c-family/ChangeLog:

PR preprocessor/105608
* c-pch.cc (c_common_read_pc

Re: [PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

On Fri, Jan 26, 2024 at 04:16:54PM -0500, Jason Merrill wrote:
> On 12/5/23 20:52, Lewis Hyatt wrote:
> > Hello-
> > 
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608
> > 
> > There are two related issues here really, a regression since GCC 11 where we
> > can ICE after restoring a PCH, and a deeper issue with bogus locations
> > assigned to macros that were defined prior to restoring a PCH.  This patch
> > fixes the ICE regression with a simple change, and I think it's appropriate
> > for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, but
> > not generally causing an ICE, and mostly affecting only the output of
> > -Wunused-macros) are not as problematic, and will be harder to fix. I could
> > take a stab at that for GCC 15. In the meantime the patch adds XFAILed
> > tests for the wrong locations (as well as passing tests for the regression
> > fix). Does it look OK please? Bootstrap + regtest all languages on x86-64
> > Linux. Thanks!
> 
> OK for trunk and branches, thanks!
>

Thanks for the review! That is all taken care of. I have one more request if
you don't mind please... There have been some further comments on the PR
indicating that the new xfailed testcase I added is failing in an unexpected
way on at least one architecture. To recap, the idea here was that

1) libcpp needs new logic to be able to output correct locations for this
case. That will be some new code that is suitable for stage 1, not now.

2) In the meantime, we fixed things up enough to avoid an ICE that showed up
in GCC 11, and added an xfailed testcase to remind about #1.

The problem is that, the reason that libcpp outputs the wrong locations, is
that it has always used a location from the old line_map instance to index
into the new line_map instance, and so the exact details of the wrong
locations it outputs depend on the state of those two line maps, which may
differ depending on system includes and things like that. So I was hoping to
make one further one-line change to libcpp, not yet to output correct
locations, but at least to output one which is the same always and doesn't
depend on random things. This would assign all restored macros to a
consistent location, one line following the #include that triggered the PCH
process. I think this probably shouldn't be backported but it would be nice
to get into GCC 14, while nothing critical, at least it would avoid the new
test failure that's being reported. But more generally, I think using a
location from a totally different line map is dangerous and could have worse
consequences that haven't been seen yet. Does it look OK please? Thanks!

-Lewis
[PATCH] libcpp: Stabilize the location for macros restored after PCH [PR105608]

libcpp currently lacks the infrastructure to assign correct locations to
macros that were defined prior to loading a PCH and then restored
afterwards. While I plan to address that for GCC 15, this one-line patch
improves things by using at least a valid location, even if it's not the
best one. Without this change, libcpp uses pfile->directive_line as the
location for the restored macros, but this location_t applies to the old
line map, not the one that was just restored from the PCH, so the resulting
location is unpredictable and depends on what was stored in the line maps
before. With this change, all restored macros get assigned locations at
line_table->highest_line, which is the first line after the #include that
triggered the PCH restore. A future patch will store the actual file name
and line number of the definition and then synthesize locations in the new
line map pointing to the right place.

libcpp/ChangeLog:

PR preprocessor/105608
* pch.cc (cpp_read_state): Set a valid location for restored
macros.

gcc/testsuite/ChangeLog:

PR preprocessor/105608
* g++.dg/pch/line-map-3.C: Adjust to expect the new location.

diff --git a/libcpp/pch.cc b/libcpp/pch.cc
index e156fe257b3..f2f74ed6ea9 100644
--- a/libcpp/pch.cc
+++ b/libcpp/pch.cc
@@ -838,7 +838,14 @@ cpp_read_state (cpp_reader *r, const char *name, FILE *f,
  != NULL)
{
  _cpp_clean_line (r);
- if (!_cpp_create_definition (r, h, 0))
+
+ /* ??? Using r->line_table->highest_line is not ideal here, but we
+do need to use some location that is relative to the new line
+map just loaded, not the old one that was in effect when these
+macros were lexed.  The proper fix is to remember the file name
+and line number where each macro was defined, and then add
+these locations into the new line map.  See PR105608.  */
+ if (!_cpp_create_definition (r, h, r->line_table->highest_line))
abort ();
  _cpp_pop_buffer (r)

ping: [PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

Hello-

May I please ping this small patch? Thanks
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/639467.html

-Lewis

On Wed, Dec 20, 2023 at 8:02 PM Lewis Hyatt  wrote:
>
> Hello-
>
> May I please ping this PCH patch? Thanks!
> https://gcc.gnu.org/pipermail/gcc-patches/2023-December/639467.html
>
> -Lewis
>
> On Tue, Dec 5, 2023 at 8:52 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608
> >
> > There are two related issues here really, a regression since GCC 11 where we
> > can ICE after restoring a PCH, and a deeper issue with bogus locations
> > assigned to macros that were defined prior to restoring a PCH.  This patch
> > fixes the ICE regression with a simple change, and I think it's appropriate
> > for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, but
> > not generally causing an ICE, and mostly affecting only the output of
> > -Wunused-macros) are not as problematic, and will be harder to fix. I could
> > take a stab at that for GCC 15. In the meantime the patch adds XFAILed
> > tests for the wrong locations (as well as passing tests for the regression
> > fix). Does it look OK please? Bootstrap + regtest all languages on x86-64
> > Linux. Thanks!
> >
> > -Lewis
> >
> > -- >8 --
> >
> > Users are allowed to define macros prior to restoring a precompiled header
> > file, as long as those macros are not defined (or are defined identically)
> > in the PCH.  However, the PCH restoration process destroys all the macro
> > definitions, so libcpp has to record them before restoring the PCH and then
> > redefine them afterward.
> >
> > This process does not currently assign great locations to the macros after
> > redefining them. Some work is needed to also remember the original locations
> > and get the line_maps instance in the right state (since, like all other
> > data structures, the line_maps instance is also reset after restoring a 
> > PCH).
> > The new testcase line-map-3.C contains XFAILed examples where the locations
> > are wrong.
> >
> > This patch addresses a more pressing issue, which is that we ICE in some
> > cases since GCC 11, hitting an assert in line-maps.cc. It happens if the
> > first line encountered after the PCH restore requires an LC_RENAME map, such
> > as will happen if the line is sufficiently long.  This is much easier to
> > fix, since we just need to call linemap_line_start before asking libcpp to
> > redefine the stored macros, instead of afterward, to avoid the unexpected
> > need for an LC_RENAME before an LC_ENTER has been seen.
> >
> > gcc/c-family/ChangeLog:
> >
> > PR preprocessor/105608
> > * c-pch.cc (c_common_read_pch): Start a new line map before asking
> > libcpp to restore macros defined prior to reading the PCH, instead
> > of afterward.
> >
> > gcc/testsuite/ChangeLog:
> >
> > PR preprocessor/105608
> > * g++.dg/pch/line-map-1.C: New test.
> > * g++.dg/pch/line-map-1.Hs: New test.
> > * g++.dg/pch/line-map-2.C: New test.
> > * g++.dg/pch/line-map-2.Hs: New test.
> > * g++.dg/pch/line-map-3.C: New test.
> > * g++.dg/pch/line-map-3.Hs: New test.
> > ---
> >  gcc/c-family/c-pch.cc  |  5 ++---
> >  gcc/testsuite/g++.dg/pch/line-map-1.C  |  4 
> >  gcc/testsuite/g++.dg/pch/line-map-1.Hs |  1 +
> >  gcc/testsuite/g++.dg/pch/line-map-2.C  |  6 ++
> >  gcc/testsuite/g++.dg/pch/line-map-2.Hs |  1 +
> >  gcc/testsuite/g++.dg/pch/line-map-3.C  | 23 +++
> >  gcc/testsuite/g++.dg/pch/line-map-3.Hs |  1 +
> >  7 files changed, 38 insertions(+), 3 deletions(-)
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.C
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.Hs
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.C
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.Hs
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.C
> >  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.Hs
> >
> > diff --git a/gcc/c-family/c-pch.cc b/gcc/c-family/c-pch.cc
> > index 2f014fca210..9ee6f179002 100644
> > --- a/gcc/c-family/c-pch.cc
> > +++ b/gcc/c-family/c-pch.cc
> > @@ -342,6 +342,8 @@ c_common_read_pch (cpp_reader *pfile, const char *name,
> >gt_pch_restore (f);
> >cpp_set_line_map (pfile, line_table);
> >rebuild_location_adhoc_htab (line_table);
> > +  line_table->trace_includ

ping: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

May I please ask again about this one? It's just a couple lines, and I
think it fixes an important gap in the logic for #pragma GCC
diagnostic. The PR was not reported by me so I think at least one
other person does care about it :). Thanks!

https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

-Lewis

On Mon, Jan 8, 2024 at 6:53 PM Lewis Hyatt  wrote:
>
> Can I please ping this one again? It's 3 lines or so to fix the PR. Thanks!
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
>
> On Tue, Dec 19, 2023 at 6:20 PM Lewis Hyatt  wrote:
> >
> > Hello-
> >
> > May I please ping this one? Thanks...
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
> >
> > -Lewis
> >
> > On Wed, Nov 29, 2023 at 7:05 PM Lewis Hyatt  wrote:
> > >
> > > On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> > > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> > > >
> > > > This patch fixes the behavior of `#pragma GCC diagnostic pop' for 
> > > > permissive
> > > > error diagnostics such as -Wnarrowing (in C++11). Those currently do not
> > > > return to the correct state after the last pop; they become effectively
> > > > simple warnings instead. Bootstrap + regtest all languages on x86-64, 
> > > > does
> > > > it look OK please? Thanks!
> > >
> > > Hello-
> > >
> > > May I please ping this bug fix?
> > > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html
> > >
> > > Please note, it requires a trivial rebase on top of recent changes to
> > > the class diagnostic_context public interface. I attached the rebased 
> > > patch
> > > here as well. Thanks!
> > >
> > > -Lewis

[PATCH] libcpp: Support extended characters for #pragma {push, pop}_macro [PR109704]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=109704

The below patch fixes the issue noted in the PR that extended characters
cannot appear in the identifier passed to a #pragma push_macro or #pragma
pop_macro. Bootstrap + regtest all languages on x86-64 Linux. Is it OK for
GCC 13 please?

I know we just entered stage 4, however I feel this is kinda like an old
regression, given that the issue was not apparent until support for UCNs and
UTF-8 in identifiers got added. FWIW, it would be nice if it makes it into
GCC 13, because AFAIK all other UTF-8-related bugs are fixed in this
release. (The other major one was for extended characters in a user-defined
literal, that was fixed by r14-2629).

Speaking of just entering stage 4. I do have 4 really short patches sent
over the past several months that never got any response. Is there any
chance someone may have a few minutes to look at them please? They are
really just like 1-3 line fixes for PRs.

libcpp (pinged once recently):
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/640386.html

diagnostics (pinged for 3rd time last week):
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

c-family/PCH (pinged a month ago):
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/639467.html

Thanks!

-Lewis

-- >8 --

The implementation of #pragma push_macro and #pragma pop_macro has to date
made use of an ad-hoc function, _cpp_lex_identifier(), which lexes an
identifier out of a string. When support was added for extended characters
in identifiers ($, UCNs, or UTF-8), that support was added only for the
"normal" way of lexing identifiers out of a cpp_buffer (_cpp_lex_direct) and
not for the ad-hoc way. Consequently, extended identifiers are not usable
with these pragmas.

The logic for lexing identifiers has become more complicated than it was
when _cpp_lex_identifier() was written -- it now handles things like \N{}
escapes in C++, for instance -- and it no longer seems practical to maintain
a redundant code path for lexing identifiers. Address the issue by changing
the implementation of #pragma {push,pop}_macro to lex identifiers in the
expected way, i.e. by pushing a cpp_buffer and lexing the identifier from
there.

The existing implementation has some quirks because of the ad-hoc parsing
logic. For example:

 #pragma push_macro("X ")
 ...
 #pragma pop_macro("X")

will not restore macro X (note the extra space in the first string). However:

 #pragma push_macro("X ")
 ...
 #pragma pop_macro("X ")

actually does sucessfully restore "X". This is because the key for looking
up the saved macro on the push stack is the original string passed, so the
string passed to pop_macro needs to match it exactly. It is not that easy to
reproduce this logic in the world of extended characters, given that for
example it should be valid to pass a UCN to push_macro, and the
corresponding UTF-8 to pop_macro. Given that this aspect of the existing
behavior seems unintentional and has no tests (and does not match other
implementations), I opted to make the new logic more straightforward. The
string passed needs to lex to one token, which must be a valid identifier,
or else no action is taken and no error is generated. Any diagnostics
encountered during lexing (e.g., due to a UTF-8 character not permitted to
appear in an identifier) are also suppressed.

It could be nice (for GCC 15) to also add a warning if a pop_macro does not
match a previous push_macro.

libcpp/ChangeLog:

PR preprocessor/109704
* include/cpplib.h (class cpp_auto_suppress_diagnostics): New class.
* errors.cc
(cpp_auto_suppress_diagnostics::cpp_auto_suppress_diagnostics): New
function.
(cpp_auto_suppress_diagnostics::~cpp_auto_suppress_diagnostics): New
function.
* charset.cc (noop_diagnostic_cb): Remove.
(cpp_interpret_string_ranges): Refactor diagnostic suppression logic
into new class cpp_auto_suppress_diagnostics.
(count_source_chars): Likewise.
* directives.cc (cpp_pop_definition): Add cpp_hashnode argument.
(lex_identifier_from_string): New static helper function.
(push_pop_macro_common): Refactor common logic from
do_pragma_push_macro and do_pragma_pop_macro; use
lex_identifier_from_string instead of _cpp_lex_identifier.
(do_pragma_push_macro): Reimplement using push_pop_macro_common.
(do_pragma_pop_macro): Likewise.
* internal.h (_cpp_lex_identifier): Remove.
* lex.cc (lex_identifier_intern): Remove.
(_cpp_lex_identifier): Remove.

gcc/testsuite/ChangeLog:

PR preprocessor/109704
* c-c++-common/cpp/pragma-push-pop-utf8.c: New test.
* g++.dg/pch/pushpop-2.C: New test.
* g++.dg/pch/pushpop-2.Hs: New test.
* gcc.dg/pch/pushpop-2.c: New test.
* gcc.dg/pch/pushpop-2.hs: New test.
---
 libcpp/charset.cc   

ping: [PATCH] libcpp: Fix __has_include_next ICE in the last directory of the path [PR80755]

[Lewis Hyatt]

Can I please ping this one? Thanks...
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/641247.html

-Lewis

On Thu, Dec 21, 2023 at 7:37 AM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80755
>
> Here is a short fix for the ICE in libcpp noted in the PR. Bootstrap +
> regtest all languages on x86-64 Linux. Is it OK please? Thanks!
>
> -Lewis
>
> -- >8 --
>
> In libcpp/files.cc, the function _cpp_has_header(), which implements
> __has_include and __has_include_next, does not check for a NULL return value
> from search_path_head(), leading to an ICE tripping an assert when
> _cpp_find_file() tries to use it. Fix it by checking for that case and
> silently returning false instead.
>
> As suggested by the PR author, it is easiest to make a testcase by using
> the -idirafter option. To enable that, also modify the dg-additional-options
> testsuite procedure to make the global $srcdir available, since -idirafter
> requires the full path.
>
> libcpp/ChangeLog:
>
> PR preprocessor/80755
> * files.cc (search_path_head): Add SUPPRESS_DIAGNOSTIC argument
> defaulting to false.
> (_cpp_has_header): Silently return false if the search path has been
> exhausted, rather than issuing a diagnostic and then hitting an
> assert.
>
> gcc/testsuite/ChangeLog:
>
> * lib/gcc-defs.exp (dg-additional-options): Make $srcdir usable in a
> dg-additional-options directive.
> * c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h: New 
> test.
> * c-c++-common/cpp/has-include-next-2.c: New test.
> ---
>  libcpp/files.cc  | 12 
>  .../cpp/has-include-next-2-dir/has-include-next-2.h  |  3 +++
>  gcc/testsuite/c-c++-common/cpp/has-include-next-2.c  |  4 
>  gcc/testsuite/lib/gcc-defs.exp   |  1 +
>  4 files changed, 16 insertions(+), 4 deletions(-)
>  create mode 100644 
> gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-next-2.c
>
> diff --git a/libcpp/files.cc b/libcpp/files.cc
> index 27301d79fa4..aaab4b13a6a 100644
> --- a/libcpp/files.cc
> +++ b/libcpp/files.cc
> @@ -181,7 +181,8 @@ static bool read_file_guts (cpp_reader *pfile, _cpp_file 
> *file,
>  static bool read_file (cpp_reader *pfile, _cpp_file *file,
>location_t loc);
>  static struct cpp_dir *search_path_head (cpp_reader *, const char *fname,
> -int angle_brackets, enum include_type);
> +int angle_brackets, enum 
> include_type,
> +bool suppress_diagnostic = false);
>  static const char *dir_name_of_file (_cpp_file *file);
>  static void open_file_failed (cpp_reader *pfile, _cpp_file *file, int,
>   location_t);
> @@ -1041,7 +1042,7 @@ _cpp_mark_file_once_only (cpp_reader *pfile, _cpp_file 
> *file)
> nothing left in the path, returns NULL.  */
>  static struct cpp_dir *
>  search_path_head (cpp_reader *pfile, const char *fname, int angle_brackets,
> - enum include_type type)
> + enum include_type type, bool suppress_diagnostic)
>  {
>cpp_dir *dir;
>_cpp_file *file;
> @@ -1070,7 +1071,7 @@ search_path_head (cpp_reader *pfile, const char *fname, 
> int angle_brackets,
>  return make_cpp_dir (pfile, dir_name_of_file (file),
>  pfile->buffer ? pfile->buffer->sysp : 0);
>
> -  if (dir == NULL)
> +  if (dir == NULL && !suppress_diagnostic)
>  cpp_error (pfile, CPP_DL_ERROR,
>"no include path in which to search for %s", fname);
>
> @@ -2164,7 +2165,10 @@ bool
>  _cpp_has_header (cpp_reader *pfile, const char *fname, int angle_brackets,
>  enum include_type type)
>  {
> -  cpp_dir *start_dir = search_path_head (pfile, fname, angle_brackets, type);
> +  cpp_dir *start_dir = search_path_head (pfile, fname, angle_brackets, type,
> +/* suppress_diagnostic = */ true);
> +  if (!start_dir)
> +return false;
>_cpp_file *file = _cpp_find_file (pfile, fname, start_dir, angle_brackets,
> _cpp_FFK_HAS_INCLUDE, 0);
>return file->err_no != ENOENT;
> diff --git 
> a/gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h 
> b/gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
> new file mode 100644
> index 000..1e4be6ce7a3
> --- /dev/null
> +++ 
> b/gcc/testsu

ping^3: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

Can I please ping this one again? It's 3 lines or so to fix the PR. Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

On Tue, Dec 19, 2023 at 6:20 PM Lewis Hyatt  wrote:
>
> Hello-
>
> May I please ping this one? Thanks...
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html
>
> -Lewis
>
> On Wed, Nov 29, 2023 at 7:05 PM Lewis Hyatt  wrote:
> >
> > On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> > >
> > > This patch fixes the behavior of `#pragma GCC diagnostic pop' for 
> > > permissive
> > > error diagnostics such as -Wnarrowing (in C++11). Those currently do not
> > > return to the correct state after the last pop; they become effectively
> > > simple warnings instead. Bootstrap + regtest all languages on x86-64, does
> > > it look OK please? Thanks!
> >
> > Hello-
> >
> > May I please ping this bug fix?
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html
> >
> > Please note, it requires a trivial rebase on top of recent changes to
> > the class diagnostic_context public interface. I attached the rebased patch
> > here as well. Thanks!
> >
> > -Lewis

Re: [PATCH v2] libstdc++: Add Unicode-aware width estimation for std::format

[Lewis Hyatt]

On Sat, Jan 6, 2024 at 11:40 AM Jonathan Wakely  wrote:
>
> Here's a V2 patch which addresses the two things I mentioned: the new
> Python script now generates a complete file that can just be included by
> , and the full Unicode 15.1.0 grapheme cluster break
> rules are supported (I think ... more testing needed for some of the
> complex rules).
>
> -- >8 --

Thanks, by the way, for fixing the typo in gen_wcwidth.py.
One thing I wanted to point out, the file contrib/unicode/README
contains a list of steps to follow in order to update to a new Unicode
version. There are 10 or so steps to generate everything libcpp and
diagnostics care about. Do you think it's worth adding something for
the new libstdc++ parts there too? I guess it may not be desirable to
update them always at the same time though.

-Lewis

ping: [PATCH] libcpp: Fix macro expansion for argument of __has_include [PR110558]

[Lewis Hyatt]

Hello-

May I please ping this one? Thanks...

https://gcc.gnu.org/pipermail/gcc-patches/2023-December/640386.html

-Lewis

On Tue, Dec 12, 2023 at 6:18 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110558
>
> This is a small fix for the libcpp issue noted in the PR. Bootstrap +
> regtest all languages on x86-64 Linux. Is it ok for trunk please?
>
> Also, it's not a regression, having never worked since __has_include was
> introduced in GCC 5, but FWIW the fix would backport fine to all branches
> since then... so I think backport to 11,12,13 would make sense assuming the
> patch is OK. Thanks!
>
> -Lewis
>
> -- >8 --
>
> When the file name for a #include directive is the result of stringifying a
> macro argument, libcpp needs to take some care to get the whitespace
> correct; in particular stringify_arg() needs to see a CPP_PADDING token
> between macro tokens so that it can figure out when to output space between
> tokens. The CPP_PADDING tokens are not normally generated when handling a
> preprocessor directive, but for #include-like directives, libcpp sets the
> state variable pfile->state.directive_wants_padding to TRUE so that the
> CPP_PADDING tokens will be output, and then everything works fine for
> computed includes.
>
> As the PR points out, things do not work fine for __has_include. Fix that by
> setting the state variable the same as is done for #include.
>
> libcpp/ChangeLog:
>
> PR preprocessor/110558
> * macro.cc (builtin_has_include): Set
> pfile->state.directive_wants_padding prior to lexing the
> file name, in case it comes from macro expansion.
>
> gcc/testsuite/ChangeLog:
>
> PR preprocessor/110558
> * c-c++-common/cpp/has-include-2.c: New test.
> * c-c++-common/cpp/has-include-2.h: New test.
> ---
>  libcpp/macro.cc|  3 +++
>  gcc/testsuite/c-c++-common/cpp/has-include-2.c | 12 
>  gcc/testsuite/c-c++-common/cpp/has-include-2.h |  1 +
>  3 files changed, 16 insertions(+)
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-2.c
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-2.h
>
> diff --git a/libcpp/macro.cc b/libcpp/macro.cc
> index 6f24a9d6f3a..15140c60023 100644
> --- a/libcpp/macro.cc
> +++ b/libcpp/macro.cc
> @@ -398,6 +398,8 @@ builtin_has_include (cpp_reader *pfile, cpp_hashnode *op, 
> bool has_next)
>NODE_NAME (op));
>
>pfile->state.angled_headers = true;
> +  const auto sav_padding = pfile->state.directive_wants_padding;
> +  pfile->state.directive_wants_padding = true;
>const cpp_token *token = cpp_get_token_no_padding (pfile);
>bool paren = token->type == CPP_OPEN_PAREN;
>if (paren)
> @@ -406,6 +408,7 @@ builtin_has_include (cpp_reader *pfile, cpp_hashnode *op, 
> bool has_next)
>  cpp_error (pfile, CPP_DL_ERROR,
>"missing '(' before \"%s\" operand", NODE_NAME (op));
>pfile->state.angled_headers = false;
> +  pfile->state.directive_wants_padding = sav_padding;
>
>bool bracket = token->type != CPP_STRING;
>char *fname = NULL;
> diff --git a/gcc/testsuite/c-c++-common/cpp/has-include-2.c 
> b/gcc/testsuite/c-c++-common/cpp/has-include-2.c
> new file mode 100644
> index 000..5cd00cb3fb5
> --- /dev/null
> +++ b/gcc/testsuite/c-c++-common/cpp/has-include-2.c
> @@ -0,0 +1,12 @@
> +/* PR preprocessor/110558 */
> +/* { dg-do preprocess } */
> +#define STRINGIZE(x) #x
> +#define GET_INCLUDE(i) STRINGIZE(has-include-i.h)
> +/* Spaces surrounding the macro args previously caused a problem for 
> __has_include().  */
> +#if __has_include(GET_INCLUDE(2)) && __has_include(GET_INCLUDE( 2)) && 
> __has_include(GET_INCLUDE( 2 ))
> +#include GET_INCLUDE(2)
> +#include GET_INCLUDE( 2)
> +#include GET_INCLUDE( 2 )
> +#else
> +#error "__has_include did not handle padding properly" /* { dg-bogus 
> "__has_include" } */
> +#endif
> diff --git a/gcc/testsuite/c-c++-common/cpp/has-include-2.h 
> b/gcc/testsuite/c-c++-common/cpp/has-include-2.h
> new file mode 100644
> index 000..57c402b32a8
> --- /dev/null
> +++ b/gcc/testsuite/c-c++-common/cpp/has-include-2.h
> @@ -0,0 +1 @@
> +/* PR preprocessor/110558 */

[PATCH] libcpp: Fix __has_include_next ICE in the last directory of the path [PR80755]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80755

Here is a short fix for the ICE in libcpp noted in the PR. Bootstrap +
regtest all languages on x86-64 Linux. Is it OK please? Thanks!

-Lewis

-- >8 --

In libcpp/files.cc, the function _cpp_has_header(), which implements
__has_include and __has_include_next, does not check for a NULL return value
from search_path_head(), leading to an ICE tripping an assert when
_cpp_find_file() tries to use it. Fix it by checking for that case and
silently returning false instead.

As suggested by the PR author, it is easiest to make a testcase by using
the -idirafter option. To enable that, also modify the dg-additional-options
testsuite procedure to make the global $srcdir available, since -idirafter
requires the full path.

libcpp/ChangeLog:

PR preprocessor/80755
* files.cc (search_path_head): Add SUPPRESS_DIAGNOSTIC argument
defaulting to false.
(_cpp_has_header): Silently return false if the search path has been
exhausted, rather than issuing a diagnostic and then hitting an
assert.

gcc/testsuite/ChangeLog:

* lib/gcc-defs.exp (dg-additional-options): Make $srcdir usable in a
dg-additional-options directive.
* c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h: New 
test.
* c-c++-common/cpp/has-include-next-2.c: New test.
---
 libcpp/files.cc  | 12 
 .../cpp/has-include-next-2-dir/has-include-next-2.h  |  3 +++
 gcc/testsuite/c-c++-common/cpp/has-include-next-2.c  |  4 
 gcc/testsuite/lib/gcc-defs.exp   |  1 +
 4 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 
gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
 create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-next-2.c

diff --git a/libcpp/files.cc b/libcpp/files.cc
index 27301d79fa4..aaab4b13a6a 100644
--- a/libcpp/files.cc
+++ b/libcpp/files.cc
@@ -181,7 +181,8 @@ static bool read_file_guts (cpp_reader *pfile, _cpp_file 
*file,
 static bool read_file (cpp_reader *pfile, _cpp_file *file,
   location_t loc);
 static struct cpp_dir *search_path_head (cpp_reader *, const char *fname,
-int angle_brackets, enum include_type);
+int angle_brackets, enum include_type,
+bool suppress_diagnostic = false);
 static const char *dir_name_of_file (_cpp_file *file);
 static void open_file_failed (cpp_reader *pfile, _cpp_file *file, int,
  location_t);
@@ -1041,7 +1042,7 @@ _cpp_mark_file_once_only (cpp_reader *pfile, _cpp_file 
*file)
nothing left in the path, returns NULL.  */
 static struct cpp_dir *
 search_path_head (cpp_reader *pfile, const char *fname, int angle_brackets,
- enum include_type type)
+ enum include_type type, bool suppress_diagnostic)
 {
   cpp_dir *dir;
   _cpp_file *file;
@@ -1070,7 +1071,7 @@ search_path_head (cpp_reader *pfile, const char *fname, 
int angle_brackets,
 return make_cpp_dir (pfile, dir_name_of_file (file),
 pfile->buffer ? pfile->buffer->sysp : 0);
 
-  if (dir == NULL)
+  if (dir == NULL && !suppress_diagnostic)
 cpp_error (pfile, CPP_DL_ERROR,
   "no include path in which to search for %s", fname);
 
@@ -2164,7 +2165,10 @@ bool
 _cpp_has_header (cpp_reader *pfile, const char *fname, int angle_brackets,
 enum include_type type)
 {
-  cpp_dir *start_dir = search_path_head (pfile, fname, angle_brackets, type);
+  cpp_dir *start_dir = search_path_head (pfile, fname, angle_brackets, type,
+/* suppress_diagnostic = */ true);
+  if (!start_dir)
+return false;
   _cpp_file *file = _cpp_find_file (pfile, fname, start_dir, angle_brackets,
_cpp_FFK_HAS_INCLUDE, 0);
   return file->err_no != ENOENT;
diff --git 
a/gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h 
b/gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
new file mode 100644
index 000..1e4be6ce7a3
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/has-include-next-2-dir/has-include-next-2.h
@@ -0,0 +1,3 @@
+#if __has_include_next()
+/* This formerly led to an ICE when the current directory was the last one in 
the path.  */
+#endif
diff --git a/gcc/testsuite/c-c++-common/cpp/has-include-next-2.c 
b/gcc/testsuite/c-c++-common/cpp/has-include-next-2.c
new file mode 100644
index 000..4928d3e992c
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/has-include-next-2.c
@@ -0,0 +1,4 @@
+/* PR preprocessor/80755 */
+/* { dg-do preprocess } */
+/* { dg-additional-options "-idirafter 
$srcdir/c-c++-common/cpp/has-include-next-2-dir" } */
+#include 
diff --git a/gcc/testsuite/lib/gcc-defs.exp b/gcc/testsuite/lib/gcc-defs.exp
index fc

Re: [PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

Hello-

May I please ping this PCH patch? Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2023-December/639467.html

-Lewis

On Tue, Dec 5, 2023 at 8:52 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608
>
> There are two related issues here really, a regression since GCC 11 where we
> can ICE after restoring a PCH, and a deeper issue with bogus locations
> assigned to macros that were defined prior to restoring a PCH.  This patch
> fixes the ICE regression with a simple change, and I think it's appropriate
> for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, but
> not generally causing an ICE, and mostly affecting only the output of
> -Wunused-macros) are not as problematic, and will be harder to fix. I could
> take a stab at that for GCC 15. In the meantime the patch adds XFAILed
> tests for the wrong locations (as well as passing tests for the regression
> fix). Does it look OK please? Bootstrap + regtest all languages on x86-64
> Linux. Thanks!
>
> -Lewis
>
> -- >8 --
>
> Users are allowed to define macros prior to restoring a precompiled header
> file, as long as those macros are not defined (or are defined identically)
> in the PCH.  However, the PCH restoration process destroys all the macro
> definitions, so libcpp has to record them before restoring the PCH and then
> redefine them afterward.
>
> This process does not currently assign great locations to the macros after
> redefining them. Some work is needed to also remember the original locations
> and get the line_maps instance in the right state (since, like all other
> data structures, the line_maps instance is also reset after restoring a PCH).
> The new testcase line-map-3.C contains XFAILed examples where the locations
> are wrong.
>
> This patch addresses a more pressing issue, which is that we ICE in some
> cases since GCC 11, hitting an assert in line-maps.cc. It happens if the
> first line encountered after the PCH restore requires an LC_RENAME map, such
> as will happen if the line is sufficiently long.  This is much easier to
> fix, since we just need to call linemap_line_start before asking libcpp to
> redefine the stored macros, instead of afterward, to avoid the unexpected
> need for an LC_RENAME before an LC_ENTER has been seen.
>
> gcc/c-family/ChangeLog:
>
> PR preprocessor/105608
> * c-pch.cc (c_common_read_pch): Start a new line map before asking
> libcpp to restore macros defined prior to reading the PCH, instead
> of afterward.
>
> gcc/testsuite/ChangeLog:
>
> PR preprocessor/105608
> * g++.dg/pch/line-map-1.C: New test.
> * g++.dg/pch/line-map-1.Hs: New test.
> * g++.dg/pch/line-map-2.C: New test.
> * g++.dg/pch/line-map-2.Hs: New test.
> * g++.dg/pch/line-map-3.C: New test.
> * g++.dg/pch/line-map-3.Hs: New test.
> ---
>  gcc/c-family/c-pch.cc  |  5 ++---
>  gcc/testsuite/g++.dg/pch/line-map-1.C  |  4 
>  gcc/testsuite/g++.dg/pch/line-map-1.Hs |  1 +
>  gcc/testsuite/g++.dg/pch/line-map-2.C  |  6 ++
>  gcc/testsuite/g++.dg/pch/line-map-2.Hs |  1 +
>  gcc/testsuite/g++.dg/pch/line-map-3.C  | 23 +++
>  gcc/testsuite/g++.dg/pch/line-map-3.Hs |  1 +
>  7 files changed, 38 insertions(+), 3 deletions(-)
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.C
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.Hs
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.C
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.Hs
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.C
>  create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.Hs
>
> diff --git a/gcc/c-family/c-pch.cc b/gcc/c-family/c-pch.cc
> index 2f014fca210..9ee6f179002 100644
> --- a/gcc/c-family/c-pch.cc
> +++ b/gcc/c-family/c-pch.cc
> @@ -342,6 +342,8 @@ c_common_read_pch (cpp_reader *pfile, const char *name,
>gt_pch_restore (f);
>cpp_set_line_map (pfile, line_table);
>rebuild_location_adhoc_htab (line_table);
> +  line_table->trace_includes = saved_trace_includes;
> +  linemap_add (line_table, LC_ENTER, 0, saved_loc.file, saved_loc.line);
>
>timevar_push (TV_PCH_CPP_RESTORE);
>if (cpp_read_state (pfile, name, f, smd) != 0)
> @@ -355,9 +357,6 @@ c_common_read_pch (cpp_reader *pfile, const char *name,
>
>fclose (f);
>
> -  line_table->trace_includes = saved_trace_includes;
> -  linemap_add (line_table, LC_ENTER, 0, saved_loc.file, saved_loc.line);
> -
>/* Give the front end a chance to take action after a PCH file has
>   been loaded.  */
>if (lang_post_pch_load)
> diff --git a/gcc/testsuite/g++.dg/pch/line-map

ping^2: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

Hello-

May I please ping this one? Thanks...
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/638692.html

-Lewis

On Wed, Nov 29, 2023 at 7:05 PM Lewis Hyatt  wrote:
>
> On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> >
> > This patch fixes the behavior of `#pragma GCC diagnostic pop' for permissive
> > error diagnostics such as -Wnarrowing (in C++11). Those currently do not
> > return to the correct state after the last pop; they become effectively
> > simple warnings instead. Bootstrap + regtest all languages on x86-64, does
> > it look OK please? Thanks!
>
> Hello-
>
> May I please ping this bug fix?
> https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html
>
> Please note, it requires a trivial rebase on top of recent changes to
> the class diagnostic_context public interface. I attached the rebased patch
> here as well. Thanks!
>
> -Lewis

[PATCH] libcpp: Fix macro expansion for argument of __has_include [PR110558]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110558

This is a small fix for the libcpp issue noted in the PR. Bootstrap +
regtest all languages on x86-64 Linux. Is it ok for trunk please?

Also, it's not a regression, having never worked since __has_include was
introduced in GCC 5, but FWIW the fix would backport fine to all branches
since then... so I think backport to 11,12,13 would make sense assuming the
patch is OK. Thanks!

-Lewis

-- >8 --

When the file name for a #include directive is the result of stringifying a
macro argument, libcpp needs to take some care to get the whitespace
correct; in particular stringify_arg() needs to see a CPP_PADDING token
between macro tokens so that it can figure out when to output space between
tokens. The CPP_PADDING tokens are not normally generated when handling a
preprocessor directive, but for #include-like directives, libcpp sets the
state variable pfile->state.directive_wants_padding to TRUE so that the
CPP_PADDING tokens will be output, and then everything works fine for
computed includes.

As the PR points out, things do not work fine for __has_include. Fix that by
setting the state variable the same as is done for #include.

libcpp/ChangeLog:

PR preprocessor/110558
* macro.cc (builtin_has_include): Set
pfile->state.directive_wants_padding prior to lexing the
file name, in case it comes from macro expansion.

gcc/testsuite/ChangeLog:

PR preprocessor/110558
* c-c++-common/cpp/has-include-2.c: New test.
* c-c++-common/cpp/has-include-2.h: New test.
---
 libcpp/macro.cc|  3 +++
 gcc/testsuite/c-c++-common/cpp/has-include-2.c | 12 
 gcc/testsuite/c-c++-common/cpp/has-include-2.h |  1 +
 3 files changed, 16 insertions(+)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-2.c
 create mode 100644 gcc/testsuite/c-c++-common/cpp/has-include-2.h

diff --git a/libcpp/macro.cc b/libcpp/macro.cc
index 6f24a9d6f3a..15140c60023 100644
--- a/libcpp/macro.cc
+++ b/libcpp/macro.cc
@@ -398,6 +398,8 @@ builtin_has_include (cpp_reader *pfile, cpp_hashnode *op, 
bool has_next)
   NODE_NAME (op));
 
   pfile->state.angled_headers = true;
+  const auto sav_padding = pfile->state.directive_wants_padding;
+  pfile->state.directive_wants_padding = true;
   const cpp_token *token = cpp_get_token_no_padding (pfile);
   bool paren = token->type == CPP_OPEN_PAREN;
   if (paren)
@@ -406,6 +408,7 @@ builtin_has_include (cpp_reader *pfile, cpp_hashnode *op, 
bool has_next)
 cpp_error (pfile, CPP_DL_ERROR,
   "missing '(' before \"%s\" operand", NODE_NAME (op));
   pfile->state.angled_headers = false;
+  pfile->state.directive_wants_padding = sav_padding;
 
   bool bracket = token->type != CPP_STRING;
   char *fname = NULL;
diff --git a/gcc/testsuite/c-c++-common/cpp/has-include-2.c 
b/gcc/testsuite/c-c++-common/cpp/has-include-2.c
new file mode 100644
index 000..5cd00cb3fb5
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/has-include-2.c
@@ -0,0 +1,12 @@
+/* PR preprocessor/110558 */
+/* { dg-do preprocess } */
+#define STRINGIZE(x) #x
+#define GET_INCLUDE(i) STRINGIZE(has-include-i.h)
+/* Spaces surrounding the macro args previously caused a problem for 
__has_include().  */
+#if __has_include(GET_INCLUDE(2)) && __has_include(GET_INCLUDE( 2)) && 
__has_include(GET_INCLUDE( 2 ))
+#include GET_INCLUDE(2)
+#include GET_INCLUDE( 2)
+#include GET_INCLUDE( 2 )
+#else
+#error "__has_include did not handle padding properly" /* { dg-bogus 
"__has_include" } */
+#endif
diff --git a/gcc/testsuite/c-c++-common/cpp/has-include-2.h 
b/gcc/testsuite/c-c++-common/cpp/has-include-2.h
new file mode 100644
index 000..57c402b32a8
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/has-include-2.h
@@ -0,0 +1 @@
+/* PR preprocessor/110558 */

[PATCH] c-family: Fix ICE with large column number after restoring a PCH [PR105608]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105608

There are two related issues here really, a regression since GCC 11 where we
can ICE after restoring a PCH, and a deeper issue with bogus locations
assigned to macros that were defined prior to restoring a PCH.  This patch
fixes the ICE regression with a simple change, and I think it's appropriate
for GCC 14 as well as backport to 11, 12, 13. The bad locations (wrong, but
not generally causing an ICE, and mostly affecting only the output of
-Wunused-macros) are not as problematic, and will be harder to fix. I could
take a stab at that for GCC 15. In the meantime the patch adds XFAILed
tests for the wrong locations (as well as passing tests for the regression
fix). Does it look OK please? Bootstrap + regtest all languages on x86-64
Linux. Thanks!

-Lewis

-- >8 --

Users are allowed to define macros prior to restoring a precompiled header
file, as long as those macros are not defined (or are defined identically)
in the PCH.  However, the PCH restoration process destroys all the macro
definitions, so libcpp has to record them before restoring the PCH and then
redefine them afterward.

This process does not currently assign great locations to the macros after
redefining them. Some work is needed to also remember the original locations
and get the line_maps instance in the right state (since, like all other
data structures, the line_maps instance is also reset after restoring a PCH).
The new testcase line-map-3.C contains XFAILed examples where the locations
are wrong.

This patch addresses a more pressing issue, which is that we ICE in some
cases since GCC 11, hitting an assert in line-maps.cc. It happens if the
first line encountered after the PCH restore requires an LC_RENAME map, such
as will happen if the line is sufficiently long.  This is much easier to
fix, since we just need to call linemap_line_start before asking libcpp to
redefine the stored macros, instead of afterward, to avoid the unexpected
need for an LC_RENAME before an LC_ENTER has been seen.

gcc/c-family/ChangeLog:

PR preprocessor/105608
* c-pch.cc (c_common_read_pch): Start a new line map before asking
libcpp to restore macros defined prior to reading the PCH, instead
of afterward.

gcc/testsuite/ChangeLog:

PR preprocessor/105608
* g++.dg/pch/line-map-1.C: New test.
* g++.dg/pch/line-map-1.Hs: New test.
* g++.dg/pch/line-map-2.C: New test.
* g++.dg/pch/line-map-2.Hs: New test.
* g++.dg/pch/line-map-3.C: New test.
* g++.dg/pch/line-map-3.Hs: New test.
---
 gcc/c-family/c-pch.cc  |  5 ++---
 gcc/testsuite/g++.dg/pch/line-map-1.C  |  4 
 gcc/testsuite/g++.dg/pch/line-map-1.Hs |  1 +
 gcc/testsuite/g++.dg/pch/line-map-2.C  |  6 ++
 gcc/testsuite/g++.dg/pch/line-map-2.Hs |  1 +
 gcc/testsuite/g++.dg/pch/line-map-3.C  | 23 +++
 gcc/testsuite/g++.dg/pch/line-map-3.Hs |  1 +
 7 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.C
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-1.Hs
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.C
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-2.Hs
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.C
 create mode 100644 gcc/testsuite/g++.dg/pch/line-map-3.Hs

diff --git a/gcc/c-family/c-pch.cc b/gcc/c-family/c-pch.cc
index 2f014fca210..9ee6f179002 100644
--- a/gcc/c-family/c-pch.cc
+++ b/gcc/c-family/c-pch.cc
@@ -342,6 +342,8 @@ c_common_read_pch (cpp_reader *pfile, const char *name,
   gt_pch_restore (f);
   cpp_set_line_map (pfile, line_table);
   rebuild_location_adhoc_htab (line_table);
+  line_table->trace_includes = saved_trace_includes;
+  linemap_add (line_table, LC_ENTER, 0, saved_loc.file, saved_loc.line);
 
   timevar_push (TV_PCH_CPP_RESTORE);
   if (cpp_read_state (pfile, name, f, smd) != 0)
@@ -355,9 +357,6 @@ c_common_read_pch (cpp_reader *pfile, const char *name,
 
   fclose (f);
 
-  line_table->trace_includes = saved_trace_includes;
-  linemap_add (line_table, LC_ENTER, 0, saved_loc.file, saved_loc.line);
-
   /* Give the front end a chance to take action after a PCH file has
  been loaded.  */
   if (lang_post_pch_load)
diff --git a/gcc/testsuite/g++.dg/pch/line-map-1.C 
b/gcc/testsuite/g++.dg/pch/line-map-1.C
new file mode 100644
index 000..9d1ac6d1683
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/line-map-1.C
@@ -0,0 +1,4 @@
+/* PR preprocessor/105608 */
+/* { dg-do compile } */
+#define MACRO_ON_A_LONG_LINE "this line is long enough that it forces the line 
table to create an LC_RENAME map, which formerly triggered an ICE after PCH 
restore"
+#include "line-map-1.H"
diff --git a/gcc/testsuite/g++.dg/pch/line-map-1.Hs 
b/gcc/testsuite/g++.dg/pch/line-map-1.Hs
new file mode 100644
index 000..3b6178bfae0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/line-map-1.Hs
@@ -0,0 +1 @@
+/* This space intentionally le

Re: [PATCH] preprocessor: Reinitialize frontend parser after loading a PCH [PR112319]

[Lewis Hyatt]

On Thu, Nov 30, 2023 at 4:19 PM Marek Polacek  wrote:
>
> On Wed, Nov 01, 2023 at 05:54:57PM -0400, Lewis Hyatt wrote:
> > Hello-
> >
> > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112319
> >
> > This is a one-line patch to fix the GCC 14 regression noted in the
> > PR. Bootstrap + regtest all languages on x86-64 looks good. Is it OK please?
> > Thanks!
> >
> > -Lewis
> >
> > -- >8 --
> >
> > Since r14-2893, the frontend parser object needs to exist when running in
> > preprocess-only mode, because pragma_lex() is now called in that mode and
> > needs to make use of it. This is handled by calling c_init_preprocess() at
> > startup. If -fpch-preprocess is in effect (commonly, because of
> > -save-temps), a PCH file may be loaded during preprocessing, in which
> > case the parser will be destroyed, causing the issue noted in the
> > PR. Resolve it by reinitializing the frontend parser after loading the PCH.
> >
> > gcc/c-family/ChangeLog:
> >
> >   PR pch/112319
> >   * c-ppoutput.cc (cb_read_pch): Reinitialize the frontend parser
>
> "front-end"
>
> >   after loading a PCH.
> >
> > gcc/testsuite/ChangeLog:
> >
> >   PR pch/112319
> >   * g++.dg/pch/pr112319.C: New test.
> >   * g++.dg/pch/pr112319.Hs: New test.
> >   * gcc.dg/pch/pr112319.c: New test.
> >   * gcc.dg/pch/pr112319.hs: New test.
> > ---
> >  gcc/c-family/c-ppoutput.cc   | 5 +
> >  gcc/testsuite/g++.dg/pch/pr112319.C  | 5 +
> >  gcc/testsuite/g++.dg/pch/pr112319.Hs | 1 +
> >  gcc/testsuite/gcc.dg/pch/pr112319.c  | 5 +
> >  gcc/testsuite/gcc.dg/pch/pr112319.hs | 1 +
> >  5 files changed, 17 insertions(+)
> >  create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.C
> >  create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.Hs
> >  create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.c
> >  create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.hs
> >
> > diff --git a/gcc/c-family/c-ppoutput.cc b/gcc/c-family/c-ppoutput.cc
> > index 4aa2bef2c0f..4f973767976 100644
> > --- a/gcc/c-family/c-ppoutput.cc
> > +++ b/gcc/c-family/c-ppoutput.cc
> > @@ -862,4 +862,9 @@ cb_read_pch (cpp_reader *pfile, const char *name,
> >
> >fprintf (print.outf, "#pragma GCC pch_preprocess \"%s\"\n", name);
> >print.src_line++;
> > +
> > +  /* The process of reading the PCH has destroyed the frontend parser,
>
> "front-end"
>
> > + so ask the frontend to reinitialize it, in case we need it to
>
> "front end"
>
> (sorry to be overly pedantic...)
>
> Patch looks fine to me; please go ahead if you haven't pushed it already.
>

Thanks for the review! I did push it a few days ago as Jakub approved
it... I will spell front end correctly next time :).

-Lewis

ping: [PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

On Thu, Nov 09, 2023 at 04:16:10PM -0500, Lewis Hyatt wrote:
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918
> 
> This patch fixes the behavior of `#pragma GCC diagnostic pop' for permissive
> error diagnostics such as -Wnarrowing (in C++11). Those currently do not
> return to the correct state after the last pop; they become effectively
> simple warnings instead. Bootstrap + regtest all languages on x86-64, does
> it look OK please? Thanks!

Hello-

May I please ping this bug fix?
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/635871.html

Please note, it requires a trivial rebase on top of recent changes to
the class diagnostic_context public interface. I attached the rebased patch
here as well. Thanks!

-Lewis
When a diagnostic pragma changes the classification of a given diagnostic,
the global options flags (such as warn_narrowing, etc.) may get changed too.
Specifically, if a warning was not enabled initially and was later enabled
by a pragma, then the corresponding global flag will change from false to
true when the pragma is processed. That change is permanent and is not
undone by a subsequent `#pragma GCC diagnostic pop'; the warning flag needs
to remain enabled since a diagnostic could be generated later on for a
source location prior to the pop.

So in order to support popping to the initial classification, given that the
global options flags no longer reflect that state, the diagnostic_context
object itself remembers the way things were before it changed anything. The
current implementation works fine for diagnostics that are always errors or
always warnings, but it doesn't do the right thing for diagnostics that
could be either, such as -Wnarrowing. The classification of that diagnostic
(or any permerror diagnostic) depends on the state of -fpermissive; for the
particular case of -Wnarrowing it also matters whether a compile-time or
run-time narrowing is being diagnosed.

The problem is that the current implementation insists on recording whether
an enabled diagnostic should be a DK_WARNING or a DK_ERROR, and then, after
popping to the initial state, it overrides it always to that type only. Fix
that up by adding a new internal diagnostic type DK_ANY. This just indicates
that the diagnostic is enabled without mandating exactly what type of
diagnostic it should be. Then the diagnostic can be emitted with whatever
type the frontend asks for.

Incidentally, while making this change, I noticed that classify_diagnostic()
spends some time computing a return value (the old classification kind) that
is not used anywhere. The computed value seems to have some problems, mainly
that it does not take into account `#pragma GCC diagnostic pop' at all, and
so the returned value doesn't seem like it could make sense in many
contexts. Given it would also not be desirable to leak the new internal-only
DK_ANY type to outside callers, I think it would make sense in a subsequent
cleanup patch to remove the return value altogether.

gcc/ChangeLog:

PR c++/111918
* diagnostic-core.h (enum diagnostic_t): Add DK_ANY special flag.
* diagnostic.cc (diagnostic_option_classifier::classify_diagnostic):
Make use of DK_ANY to indicate a diagnostic was initially enabled.
(diagnostic_context::diagnostic_enabled): Do not change the type of
a diagnostic if the saved classification is type DK_ANY.

gcc/testsuite/ChangeLog:

PR c++/111918
* g++.dg/cpp0x/Wnarrowing21a.C: New test.
* g++.dg/cpp0x/Wnarrowing21b.C: New test.
* g++.dg/cpp0x/Wnarrowing21c.C: New test.
* g++.dg/cpp0x/Wnarrowing21d.C: New test.

diff --git a/gcc/diagnostic-core.h b/gcc/diagnostic-core.h
index 04eba3d140e..4926c48da96 100644
--- a/gcc/diagnostic-core.h
+++ b/gcc/diagnostic-core.h
@@ -33,7 +33,10 @@ typedef enum
   DK_LAST_DIAGNOSTIC_KIND,
   /* This is used for tagging pragma pops in the diagnostic
  classification history chain.  */
-  DK_POP
+  DK_POP,
+  /* This is used internally to note that a diagnostic is enabled
+ without mandating any specific type.  */
+  DK_ANY,
 } diagnostic_t;
 
 /* RAII-style class for grouping related diagnostics.  */
diff --git a/gcc/diagnostic.cc b/gcc/diagnostic.cc
index 4f66fa6acaa..fd40018a734 100644
--- a/gcc/diagnostic.cc
+++ b/gcc/diagnostic.cc
@@ -1136,8 +1136,7 @@ classify_diagnostic (const diagnostic_context *context,
   if (old_kind == DK_UNSPECIFIED)
{
  old_kind = !context->option_enabled_p (option_index)
-   ? DK_IGNORED : (context->warning_as_error_requested_p ()
-   ? DK_ERROR : DK_WARNING);
+   ? DK_IGNORED : DK_ANY;
  m_classify_diagnostic[option_index] = old_kind;
}
 
@@ -1472,7 +1471,15 @@ diagnostic_context::diagnostic_enabled (diagnostic_info 
*diagnostic)
  option.  */
   if (diag_class == DK_UNSPECIFIED
   && !option_uns

[PATCH] libcpp: Fix unsigned promotion for unevaluated divide by zero [PR112701]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112701

Here is a one-line fix to an edge case in libcpp's expression evaluator
noted in the PR. Bootstrap + regtest all languages on x86-64 Linux. Is it OK
please? Thanks!

-Lewis

-- >8 --

When libcpp encounters a divide by zero while processing a constant
expression "x/y", it returns "x" as a fallback. The value of the fallback is
not normally important, since an error will be generated anyway, but if the
expression appears in an unevaluated context, such as "0 ? 0/0u : -1", then
there will be no error, and the fallback value will be meaningful to the
extent that it may cause promotion from signed to unsigned of an operand
encountered later. As the PR notes, libcpp does not do the unsigned
promotion correctly in this case; fix it by making the fallback return value
unsigned as necessary.

libcpp/ChangeLog:

PR preprocessor/112701
* expr.cc (num_div_op): Set unsignedp appropriately when returning a
stub value for divide by 0.

gcc/testsuite/ChangeLog:

PR preprocessor/112701
* gcc.dg/cpp/expr.c: Add additional tests to cover divide by 0 in an
unevaluated context, where the unsignedness still matters.
---
 libcpp/expr.cc  |  1 +
 gcc/testsuite/gcc.dg/cpp/expr.c | 22 --
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/libcpp/expr.cc b/libcpp/expr.cc
index 825d2c2369d..4f4a9722ac7 100644
--- a/libcpp/expr.cc
+++ b/libcpp/expr.cc
@@ -2216,6 +2216,7 @@ num_div_op (cpp_reader *pfile, cpp_num lhs, cpp_num rhs, 
enum cpp_ttype op,
   if (!pfile->state.skip_eval)
cpp_error_with_line (pfile, CPP_DL_ERROR, location, 0,
 "division by zero in #if");
+  lhs.unsignedp = unsignedp;
   return lhs;
 }
 
diff --git a/gcc/testsuite/gcc.dg/cpp/expr.c b/gcc/testsuite/gcc.dg/cpp/expr.c
index 532bd681237..055e17ae753 100644
--- a/gcc/testsuite/gcc.dg/cpp/expr.c
+++ b/gcc/testsuite/gcc.dg/cpp/expr.c
@@ -1,6 +1,7 @@
 /* Copyright (C) 2000, 2001 Free Software Foundation, Inc.  */
 
 /* { dg-do preprocess } */
+/* { dg-additional-options "-Wall" } */
 
 /* Test we get signedness of ?: operator correct.  We would skip
evaluation of one argument, and might therefore not transfer its
@@ -8,10 +9,27 @@
 
 /* Neil Booth, 19 Jul 2002.  */
 
-#if (1 ? -2: 0 + 1U) < 0
+#if (1 ? -2: 0 + 1U) < 0 /* { dg-warning {the left operand of ":" changes 
sign} } */
 #error /* { dg-bogus "error" } */
 #endif
 
-#if (0 ? 0 + 1U: -2) < 0
+#if (0 ? 0 + 1U: -2) < 0 /* { dg-warning {the right operand of ":" changes 
sign} } */
 #error /* { dg-bogus "error" } */
 #endif
+
+/* PR preprocessor/112701 */
+#if (0 ? 0/0u : -1) < 0 /* { dg-warning {the right operand of ":" changes 
sign} } */
+#error /* { dg-bogus "error" } */
+#endif
+
+#if (0 ? 0u/0 : -1) < 0 /* { dg-warning {the right operand of ":" changes 
sign} } */
+#error /* { dg-bogus "error" } */
+#endif
+
+#if (1 ? -1 : 0/0u) < 0 /* { dg-warning {the left operand of ":" changes sign} 
} */
+#error /* { dg-bogus "error" } */
+#endif
+
+#if (1 ? -1 : 0u/0) < 0 /* { dg-warning {the left operand of ":" changes sign} 
} */
+#error /* { dg-bogus "error" } */
+#endif

[PATCH] Makefile.tpl: Avoid race condition in generating site.exp from the top level

[Lewis Hyatt]

Hello-

I often find it convenient to run a new c-c++-common test from the
main build dir like:

$ make -j 2 RUNTESTFLAGS=dg.exp=new-test.c check-gcc-{c,c++}

I noticed that sometimes this produces a corrupted site.exp and then no
tests work until it is remade manually. To avoid the issue, it is necessary
to do "cd gcc; make site.exp" before running a parallel make from the top
level directory. The below patch fixes it by just making that dependency on
site.exp explicit in the top level Makefile. Is it OK please? Thanks...

-Lewis

-- >8 --

A command like "make -j 2 check-gcc-c check-gcc-c++" run in the top level of
a fresh build directory does not work reliably. That will spawn two
independent make processes inside the "gcc" directory, and each of those
will attempt to create site.exp if it doesn't exist and will interfere with
each other, producing often a corrupted or empty site.exp. Resolve that by
making these targets depend on a new phony target which makes sure site.exp
is created first before starting the recursive makes.

ChangeLog:

* Makefile.in: Regenerate.
* Makefile.tpl: Add dependency on site.exp to check-gcc-* targets
---
 Makefile.in  | 30 +++---
 Makefile.tpl | 10 +-
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/Makefile.tpl b/Makefile.tpl
index 8b7783bb4f1..6e22adecd2f 100644
--- a/Makefile.tpl
+++ b/Makefile.tpl
@@ -1639,9 +1639,17 @@ cross: all-build all-gas all-ld
 @endif gcc-no-bootstrap
 
 @if gcc
+
+.PHONY: gcc-site.exp
+gcc-site.exp:
+   r=`${PWD_COMMAND}`; export r; \
+   s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+   $(HOST_EXPORTS) \
+   (cd gcc && $(MAKE) $(GCC_FLAGS_TO_PASS) site.exp);
+
 [+ FOR languages +]
 .PHONY: check-gcc-[+language+] check-[+language+]
-check-gcc-[+language+]:
+check-gcc-[+language+]: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
diff --git a/Makefile.in b/Makefile.in
index b65ab4953bc..da2344b3f3d 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -62200,8 +62200,16 @@ cross: all-build all-gas all-ld
 
 @if gcc
 
+.PHONY: gcc-site.exp
+gcc-site.exp:
+   r=`${PWD_COMMAND}`; export r; \
+   s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
+   $(HOST_EXPORTS) \
+   (cd gcc && $(MAKE) $(GCC_FLAGS_TO_PASS) site.exp);
+
+
 .PHONY: check-gcc-c check-c
-check-gcc-c:
+check-gcc-c: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62209,7 +62217,7 @@ check-gcc-c:
 check-c: check-gcc-c
 
 .PHONY: check-gcc-c++ check-c++
-check-gcc-c++:
+check-gcc-c++: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62217,7 +62225,7 @@ check-gcc-c++:
 check-c++: check-gcc-c++ check-target-libstdc++-v3 check-target-libitm-c++ 
check-target-libgomp-c++
 
 .PHONY: check-gcc-fortran check-fortran
-check-gcc-fortran:
+check-gcc-fortran: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62225,7 +62233,7 @@ check-gcc-fortran:
 check-fortran: check-gcc-fortran check-target-libquadmath 
check-target-libgfortran check-target-libgomp-fortran
 
 .PHONY: check-gcc-ada check-ada
-check-gcc-ada:
+check-gcc-ada: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62233,7 +62241,7 @@ check-gcc-ada:
 check-ada: check-gcc-ada check-target-libada
 
 .PHONY: check-gcc-objc check-objc
-check-gcc-objc:
+check-gcc-objc: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62241,7 +62249,7 @@ check-gcc-objc:
 check-objc: check-gcc-objc check-target-libobjc
 
 .PHONY: check-gcc-obj-c++ check-obj-c++
-check-gcc-obj-c++:
+check-gcc-obj-c++: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62249,7 +62257,7 @@ check-gcc-obj-c++:
 check-obj-c++: check-gcc-obj-c++
 
 .PHONY: check-gcc-go check-go
-check-gcc-go:
+check-gcc-go: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62257,7 +62265,7 @@ check-gcc-go:
 check-go: check-gcc-go check-target-libgo check-gotools
 
 .PHONY: check-gcc-m2 check-m2
-check-gcc-m2:
+check-gcc-m2: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62265,7 +62273,7 @@ check-gcc-m2:
 check-m2: check-gcc-m2 check-target-libgm2
 
 .PHONY: check-gcc-d check-d
-check-gcc-d:
+check-gcc-d: gcc-site.exp
r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
$(HOST_EXPORTS) \
@@ -62273,7 

ping: [PATCH] preprocessor: Reinitialize frontend parser after loading a PCH [PR112319]

[Lewis Hyatt]

May I please ping this one? Thanks...
https://gcc.gnu.org/pipermail/gcc-patches/2023-November/634931.html

On Wed, Nov 1, 2023 at 5:55 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112319
>
> This is a one-line patch to fix the GCC 14 regression noted in the
> PR. Bootstrap + regtest all languages on x86-64 looks good. Is it OK please?
> Thanks!
>
> -Lewis
>
> -- >8 --
>
> Since r14-2893, the frontend parser object needs to exist when running in
> preprocess-only mode, because pragma_lex() is now called in that mode and
> needs to make use of it. This is handled by calling c_init_preprocess() at
> startup. If -fpch-preprocess is in effect (commonly, because of
> -save-temps), a PCH file may be loaded during preprocessing, in which
> case the parser will be destroyed, causing the issue noted in the
> PR. Resolve it by reinitializing the frontend parser after loading the PCH.
>
> gcc/c-family/ChangeLog:
>
> PR pch/112319
> * c-ppoutput.cc (cb_read_pch): Reinitialize the frontend parser
> after loading a PCH.
>
> gcc/testsuite/ChangeLog:
>
> PR pch/112319
> * g++.dg/pch/pr112319.C: New test.
> * g++.dg/pch/pr112319.Hs: New test.
> * gcc.dg/pch/pr112319.c: New test.
> * gcc.dg/pch/pr112319.hs: New test.
> ---
>  gcc/c-family/c-ppoutput.cc   | 5 +
>  gcc/testsuite/g++.dg/pch/pr112319.C  | 5 +
>  gcc/testsuite/g++.dg/pch/pr112319.Hs | 1 +
>  gcc/testsuite/gcc.dg/pch/pr112319.c  | 5 +
>  gcc/testsuite/gcc.dg/pch/pr112319.hs | 1 +
>  5 files changed, 17 insertions(+)
>  create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.C
>  create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.Hs
>  create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.c
>  create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.hs
>
> diff --git a/gcc/c-family/c-ppoutput.cc b/gcc/c-family/c-ppoutput.cc
> index 4aa2bef2c0f..4f973767976 100644
> --- a/gcc/c-family/c-ppoutput.cc
> +++ b/gcc/c-family/c-ppoutput.cc
> @@ -862,4 +862,9 @@ cb_read_pch (cpp_reader *pfile, const char *name,
>
>fprintf (print.outf, "#pragma GCC pch_preprocess \"%s\"\n", name);
>print.src_line++;
> +
> +  /* The process of reading the PCH has destroyed the frontend parser,
> + so ask the frontend to reinitialize it, in case we need it to
> + process any #pragma directives encountered while preprocessing.  */
> +  c_init_preprocess ();
>  }
> diff --git a/gcc/testsuite/g++.dg/pch/pr112319.C 
> b/gcc/testsuite/g++.dg/pch/pr112319.C
> new file mode 100644
> index 000..9e0457e8aec
> --- /dev/null
> +++ b/gcc/testsuite/g++.dg/pch/pr112319.C
> @@ -0,0 +1,5 @@
> +/* { dg-additional-options "-Wpragmas -save-temps" } */
> +#include "pr112319.H"
> +#pragma GCC diagnostic error "-Wpragmas"
> +#pragma GCC diagnostic ignored "oops" /* { dg-error "oops" } */
> +/* { dg-regexp {[^[:space:]]*: some warnings being treated as errors} } */
> diff --git a/gcc/testsuite/g++.dg/pch/pr112319.Hs 
> b/gcc/testsuite/g++.dg/pch/pr112319.Hs
> new file mode 100644
> index 000..3b6178bfae0
> --- /dev/null
> +++ b/gcc/testsuite/g++.dg/pch/pr112319.Hs
> @@ -0,0 +1 @@
> +/* This space intentionally left blank.  */
> diff --git a/gcc/testsuite/gcc.dg/pch/pr112319.c 
> b/gcc/testsuite/gcc.dg/pch/pr112319.c
> new file mode 100644
> index 000..043881463c5
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/pch/pr112319.c
> @@ -0,0 +1,5 @@
> +/* { dg-additional-options "-Wpragmas -save-temps" } */
> +#include "pr112319.h"
> +#pragma GCC diagnostic error "-Wpragmas"
> +#pragma GCC diagnostic ignored "oops" /* { dg-error "oops" } */
> +/* { dg-regexp {[^[:space:]]*: some warnings being treated as errors} } */
> diff --git a/gcc/testsuite/gcc.dg/pch/pr112319.hs 
> b/gcc/testsuite/gcc.dg/pch/pr112319.hs
> new file mode 100644
> index 000..3b6178bfae0
> --- /dev/null
> +++ b/gcc/testsuite/gcc.dg/pch/pr112319.hs
> @@ -0,0 +1 @@
> +/* This space intentionally left blank.  */

[PATCH] c-family: Let libcpp know when the compilation is for a PCH [PR9471]

[Lewis Hyatt]

Hello-

The PR may be 20 years old, but by now it only needs a one-line fix :). Is
it OK please? Bootstrapped + regtested all langauges on x86-64 Linux.
Thanks!

-Lewis

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=9471
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47857

-- >8 --

libcpp will generate diagnostics when it encounters things in the main file
that only belong in a header file, such as `#pragma once' or `#pragma GCC
system_header'. But sometimes the main file is a header file that is just
being compiled separately, e.g. to produce a C++ module or a PCH, in which
case such diagnostics should be suppressed. libcpp already has an interface
to request that, so make use of it in the C frontends to prevent libcpp from
issuing unwanted diagnostics when compiling a PCH.

gcc/c-family/ChangeLog:

PR pch/9471
PR pch/47857
* c-opts.cc (c_common_post_options): Set cpp_opts->main_search
so libcpp knows it is compiling a header file separately.

gcc/testsuite/ChangeLog:

PR pch/9471
PR pch/47857
* g++.dg/pch/main-file-warnings.C: New test.
* g++.dg/pch/main-file-warnings.Hs: New test.
* gcc.dg/pch/main-file-warnings.c: New test.
* gcc.dg/pch/main-file-warnings.hs: New test.
---
 gcc/c-family/c-opts.cc | 3 +++
 gcc/testsuite/g++.dg/pch/main-file-warnings.C  | 7 +++
 gcc/testsuite/g++.dg/pch/main-file-warnings.Hs | 3 +++
 gcc/testsuite/gcc.dg/pch/main-file-warnings.c  | 7 +++
 gcc/testsuite/gcc.dg/pch/main-file-warnings.hs | 3 +++
 5 files changed, 23 insertions(+)
 create mode 100644 gcc/testsuite/g++.dg/pch/main-file-warnings.C
 create mode 100644 gcc/testsuite/g++.dg/pch/main-file-warnings.Hs
 create mode 100644 gcc/testsuite/gcc.dg/pch/main-file-warnings.c
 create mode 100644 gcc/testsuite/gcc.dg/pch/main-file-warnings.hs

diff --git a/gcc/c-family/c-opts.cc b/gcc/c-family/c-opts.cc
index fbabd1816c1..10403c03bd6 100644
--- a/gcc/c-family/c-opts.cc
+++ b/gcc/c-family/c-opts.cc
@@ -1174,6 +1174,9 @@ c_common_post_options (const char **pfilename)
   "the %qs debug info cannot be used with "
   "pre-compiled headers",
   debug_set_names (write_symbols & ~DWARF2_DEBUG));
+ /* Let libcpp know that the main file is a header so it won't
+complain about things like #include_next and #pragma once.  */
+ cpp_opts->main_search = CMS_header;
}
   else if (write_symbols != NO_DEBUG && write_symbols != DWARF2_DEBUG)
c_common_no_more_pch ();
diff --git a/gcc/testsuite/g++.dg/pch/main-file-warnings.C 
b/gcc/testsuite/g++.dg/pch/main-file-warnings.C
new file mode 100644
index 000..a9e8b0ba9f2
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/main-file-warnings.C
@@ -0,0 +1,7 @@
+/* PR pch/9471 */
+/* PR pch/47857 */
+/* Test will fail if any warnings get issued while compiling the header into a 
PCH.  */
+#include "main-file-warnings.H"
+#pragma once /* { dg-warning "in main file" } */
+#pragma GCC system_header /* { dg-warning "outside include file" } */
+#include_next  /* { dg-warning "in primary source file" } */
diff --git a/gcc/testsuite/g++.dg/pch/main-file-warnings.Hs 
b/gcc/testsuite/g++.dg/pch/main-file-warnings.Hs
new file mode 100644
index 000..d1582bb8290
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/main-file-warnings.Hs
@@ -0,0 +1,3 @@
+#pragma once /* { dg-bogus "in main file" } */
+#pragma GCC system_header /* { dg-bogus "outside include file" } */
+#include_next  /* { dg-bogus "in primary source file" } */
diff --git a/gcc/testsuite/gcc.dg/pch/main-file-warnings.c 
b/gcc/testsuite/gcc.dg/pch/main-file-warnings.c
new file mode 100644
index 000..aedbc15f7ba
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pch/main-file-warnings.c
@@ -0,0 +1,7 @@
+/* PR pch/9471 */
+/* PR pch/47857 */
+/* Test will fail if any warnings get issued while compiling the header into a 
PCH.  */
+#include "main-file-warnings.h"
+#pragma once /* { dg-warning "in main file" } */
+#pragma GCC system_header /* { dg-warning "outside include file" } */
+#include_next  /* { dg-warning "in primary source file" } */
diff --git a/gcc/testsuite/gcc.dg/pch/main-file-warnings.hs 
b/gcc/testsuite/gcc.dg/pch/main-file-warnings.hs
new file mode 100644
index 000..d1582bb8290
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pch/main-file-warnings.hs
@@ -0,0 +1,3 @@
+#pragma once /* { dg-bogus "in main file" } */
+#pragma GCC system_header /* { dg-bogus "outside include file" } */
+#include_next  /* { dg-bogus "in primary source file" } */

[PATCH] diagnostics: Fix behavior of permerror options after diagnostic pop [PR111918]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111918

This patch fixes the behavior of `#pragma GCC diagnostic pop' for permissive
error diagnostics such as -Wnarrowing (in C++11). Those currently do not
return to the correct state after the last pop; they become effectively
simple warnings instead. Bootstrap + regtest all languages on x86-64, does
it look OK please? Thanks!

-Lewis

-- >8 --

When a diagnostic pragma changes the classification of a given diagnostic,
the global options flags (such as warn_narrowing, etc.) may get changed too.
Specifically, if a warning was not enabled initially and was later enabled
by a pragma, then the corresponding global flag will change from false to
true when the pragma is processed. That change is permanent and is not
undone by a subsequent `#pragma GCC diagnostic pop'; the warning flag needs
to remain enabled since a diagnostic could be generated later on for a
source location prior to the pop.

So in order to support popping to the initial classification, given that the
global options flags no longer reflect that state, the diagnostic_context
object itself remembers the way things were before it changed anything. The
current implementation works fine for diagnostics that are always errors or
always warnings, but it doesn't do the right thing for diagnostics that
could be either, such as -Wnarrowing. The classification of that diagnostic
(or any permerror diagnostic) depends on the state of -fpermissive; for the
particular case of -Wnarrowing it also matters whether a compile-time or
run-time narrowing is being diagnosed.

The problem is that the current implementation insists on recording whether
an enabled diagnostic should be a DK_WARNING or a DK_ERROR, and then, after
popping to the initial state, it overrides it always to that type only. Fix
that up by adding a new internal diagnostic type DK_ANY. This just indicates
that the diagnostic is enabled without mandating exactly what type of
diagnostic it should be. Then the diagnostic can be emitted with whatever
type the frontend asks for.

Incidentally, while making this change, I noticed that classify_diagnostic()
spends some time computing a return value (the old classification kind) that
is not used anywhere. The computed value seems to have some problems, mainly
that it does not take into account `#pragma GCC diagnostic pop' at all, and
so the returned value doesn't seem like it could make sense in many
contexts. Given it would also not be desirable to leak the new internal-only
DK_ANY type to outside callers, I think it would make sense in a subsequent
cleanup patch to remove the return value altogether.

gcc/ChangeLog:

PR c++/111918
* diagnostic-core.h (enum diagnostic_t): Add DK_ANY special flag.
* diagnostic.cc (diagnostic_option_classifier::classify_diagnostic):
Make use of DK_ANY to indicate a diagnostic was initially enabled.
(diagnostic_context::diagnostic_enabled): Do not change the type of
a diagnostic if the saved classification is type DK_ANY.

gcc/testsuite/ChangeLog:

PR c++/111918
* g++.dg/cpp0x/Wnarrowing21a.C: New test.
* g++.dg/cpp0x/Wnarrowing21b.C: New test.
* g++.dg/cpp0x/Wnarrowing21c.C: New test.
* g++.dg/cpp0x/Wnarrowing21d.C: New test.
---
 gcc/diagnostic-core.h  |  5 -
 gcc/diagnostic.cc  | 13 ++---
 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21a.C | 14 ++
 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21b.C |  9 +
 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21c.C |  9 +
 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21d.C |  9 +
 6 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21a.C
 create mode 100644 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21b.C
 create mode 100644 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21c.C
 create mode 100644 gcc/testsuite/g++.dg/cpp0x/Wnarrowing21d.C

diff --git a/gcc/diagnostic-core.h b/gcc/diagnostic-core.h
index 04eba3d140e..4926c48da96 100644
--- a/gcc/diagnostic-core.h
+++ b/gcc/diagnostic-core.h
@@ -33,7 +33,10 @@ typedef enum
   DK_LAST_DIAGNOSTIC_KIND,
   /* This is used for tagging pragma pops in the diagnostic
  classification history chain.  */
-  DK_POP
+  DK_POP,
+  /* This is used internally to note that a diagnostic is enabled
+ without mandating any specific type.  */
+  DK_ANY,
 } diagnostic_t;
 
 /* RAII-style class for grouping related diagnostics.  */
diff --git a/gcc/diagnostic.cc b/gcc/diagnostic.cc
index addd6606eaa..99921a10b7b 100644
--- a/gcc/diagnostic.cc
+++ b/gcc/diagnostic.cc
@@ -1126,8 +1126,7 @@ classify_diagnostic (const diagnostic_context *context,
  old_kind = !context->m_option_enabled (option_index,
 context->m_lang_mask,
 context->m_option_state)
-   ? DK_IGNORED : (context->warning_as_error_reque

[PATCH] diagnostics: pch: Remember diagnostic pragmas in a PCH [PR64117]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64117

This fixes an old PR / enhancement request. Bootstrap + regtest all
languages on x86-64 Linux. Please let me know if it looks OK? Thanks!

-Lewis

-- >8 --

As the PR points out, we do not currently record in a PCH whether any
diagnostics were reclassified by pragmas, so a header that takes care to
adjust some diagnostics for downstream users does not work as designed when
it is precompiled.  Implement that feature by adding a new interface
diagnostic_context::save_to_pch() / restore_from_pch(), which is called on
the global diagnostic context object.

This feature also exposes the need for a small tweak to the C++ frontend.
That frontend processes `#pragma GCC diagnostic push' and `#pragma GCC
diagnostic pop' pragmas twice, once while preprocessing and once again while
compiling.  In case a translation unit contains an unbalanced set of pushes
and pops, that results in twice as many leftover states as there should be.
This does not cause any problems for a single translation unit, but if the
snapshot of the state is preserved in a PCH, then it becomes observable, for
example with a setup like in the new testcase pragma-diagnostic-3.C:

t.h:

 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored...
 //no pop at end of the file

t.c

 #include "t.h"
 #pragma GCC diagnostic pop
 //expect to be at the initial state here

If t.h has been precompiled, and if the push had been processed twice at the
time the PCH was written, then the state will not be reset as expected in
t.c. Address that by having the C++ frontend reset the push/pop history
before starting its second pass.

gcc/ChangeLog:

PR pch/64117
* Makefile.in: Add tree-diagnostic.cc to GTFILES
* diagnostic.cc (diagnostic_option_classifier::init): Handle the
classification history members, which had been omitted here.
(diagnostic_option_classifier::fini): Likewise.
(diagnostic_option_classifier::classify_diagnostic): Refactor some
logic to...
(diagnostic_context::get_original_option_classification): ...this
new function.
* diagnostic.h (struct diagnostic_option_classifier::state): Declare.
(diagnostic_option_classifier::save_state_to): Declare.
(diagnostic_option_classifier::restore_from_pch): Declare.
(diagnostic_option_classifier::get_state): Declare.
(diagnostic_option_classifier::restore_state): Declare.
(diagnostic_option_classifier::free_state): Declare.
(diagnostic_context::get_original_option_classification): Declare.
(diagnostic_context::get_classifier): New accessor functions.
(diagnostic_context::save_to_pch): Declare.
(diagnostic_context::restore_from_pch): Declare.
* ggc-common.cc (gt_pch_save): Use the new PCH interface to handle
the global diagnostic context.
(gt_pch_restore): Likewise.
* tree-diagnostic.cc (struct diagnostic_option_classifier::state):
New struct.
(struct diagnostic_pch_data): New struct.
(pch): New GC root.
(diagnostic_context::save_to_pch): New function.
(diagnostic_context::restore_from_pch): New function.
(diagnostic_option_classifier::save_state_to): New function.
(diagnostic_option_classifier::restore_from_pch): New function.
(diagnostic_option_classifier::get_state): New function.
(diagnostic_option_classifier::restore_state): New function.
(diagnostic_option_classifier::free_state): New function.

gcc/cp/ChangeLog:

PR pch/64117
* parser.cc (cp_lexer_new_main): Restore the diagnostics
classifications to their original state after the first pass through
the input.

gcc/testsuite/ChangeLog:

PR pch/64117
* g++.dg/pch/pragma-diagnostic-1.C: New test.
* g++.dg/pch/pragma-diagnostic-1.Hs: New test.
* g++.dg/pch/pragma-diagnostic-2.C: New test.
* g++.dg/pch/pragma-diagnostic-2.Hs: New test.
* g++.dg/pch/pragma-diagnostic-3.C: New test.
* g++.dg/pch/pragma-diagnostic-3.Hs: New test.
* g++.dg/pch/pragma-diagnostic-4.C: New test.
* g++.dg/pch/pragma-diagnostic-4.Hs: New test.
* g++.dg/pch/pragma-diagnostic-5.C: New test.
* g++.dg/pch/pragma-diagnostic-5.Hs: New test.
* gcc.dg/pch/pragma-diagnostic-1.c: New test.
* gcc.dg/pch/pragma-diagnostic-1.hs: New test.
* gcc.dg/pch/pragma-diagnostic-2.c: New test.
* gcc.dg/pch/pragma-diagnostic-2.hs: New test.
* gcc.dg/pch/pragma-diagnostic-3.c: New test.
* gcc.dg/pch/pragma-diagnostic-3.hs: New test.
* gcc.dg/pch/pragma-diagnostic-4.c: New test.
* gcc.dg/pch/pragma-diagnostic-4.hs: New test.
---
 gcc/Makefile.in   |   1 +
 gcc/cp/parser.cc  |  11 ++
 gcc/diagnostic.cc |  26 ++-
 gcc/diagn

Re: [PATCH 1/2] libdiagnostics: header and examples

[Lewis Hyatt]

On Mon, Nov 6, 2023 at 8:29 PM David Malcolm  wrote:
>
> Here's a work-in-progress patch for GCC that adds a libdiagnostics.h
> header describing the public interface, along with various testcases
> that show usage examples for the API.  Various aspects of this need
> work; posting now for early feedback on overall direction.
>
> How does the interface look?
>
...
> +typedef unsigned int diagnostic_location_t;

One comment that occurred to me... for GCC we have a lot of PRs that
are unhappy about the 32-bit location_t and the consequent issues that
arise with very large source files, or with very long lines that lose
column information.
So far GCC has been able to get by with "don't do that" advice, but a
more general libdiagnostics may need to avoid that arbitrary
limitation? I feel like it may not be that long before GCC needs to
deal with it as well, perhaps with a configure option, but even now,
it could make sense for libdiagnostic to use a 64-bit location_t
itself from the outset, so it won't need to change later, even if it's
practically restricted to 32 bits for now.

-Lewis

[PATCH] preprocessor: Reinitialize frontend parser after loading a PCH [PR112319]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=112319

This is a one-line patch to fix the GCC 14 regression noted in the
PR. Bootstrap + regtest all languages on x86-64 looks good. Is it OK please?
Thanks!

-Lewis

-- >8 --

Since r14-2893, the frontend parser object needs to exist when running in
preprocess-only mode, because pragma_lex() is now called in that mode and
needs to make use of it. This is handled by calling c_init_preprocess() at
startup. If -fpch-preprocess is in effect (commonly, because of
-save-temps), a PCH file may be loaded during preprocessing, in which
case the parser will be destroyed, causing the issue noted in the
PR. Resolve it by reinitializing the frontend parser after loading the PCH.

gcc/c-family/ChangeLog:

PR pch/112319
* c-ppoutput.cc (cb_read_pch): Reinitialize the frontend parser
after loading a PCH.

gcc/testsuite/ChangeLog:

PR pch/112319
* g++.dg/pch/pr112319.C: New test.
* g++.dg/pch/pr112319.Hs: New test.
* gcc.dg/pch/pr112319.c: New test.
* gcc.dg/pch/pr112319.hs: New test.
---
 gcc/c-family/c-ppoutput.cc   | 5 +
 gcc/testsuite/g++.dg/pch/pr112319.C  | 5 +
 gcc/testsuite/g++.dg/pch/pr112319.Hs | 1 +
 gcc/testsuite/gcc.dg/pch/pr112319.c  | 5 +
 gcc/testsuite/gcc.dg/pch/pr112319.hs | 1 +
 5 files changed, 17 insertions(+)
 create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.C
 create mode 100644 gcc/testsuite/g++.dg/pch/pr112319.Hs
 create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.c
 create mode 100644 gcc/testsuite/gcc.dg/pch/pr112319.hs

diff --git a/gcc/c-family/c-ppoutput.cc b/gcc/c-family/c-ppoutput.cc
index 4aa2bef2c0f..4f973767976 100644
--- a/gcc/c-family/c-ppoutput.cc
+++ b/gcc/c-family/c-ppoutput.cc
@@ -862,4 +862,9 @@ cb_read_pch (cpp_reader *pfile, const char *name,
 
   fprintf (print.outf, "#pragma GCC pch_preprocess \"%s\"\n", name);
   print.src_line++;
+
+  /* The process of reading the PCH has destroyed the frontend parser,
+ so ask the frontend to reinitialize it, in case we need it to
+ process any #pragma directives encountered while preprocessing.  */
+  c_init_preprocess ();
 }
diff --git a/gcc/testsuite/g++.dg/pch/pr112319.C 
b/gcc/testsuite/g++.dg/pch/pr112319.C
new file mode 100644
index 000..9e0457e8aec
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/pr112319.C
@@ -0,0 +1,5 @@
+/* { dg-additional-options "-Wpragmas -save-temps" } */
+#include "pr112319.H"
+#pragma GCC diagnostic error "-Wpragmas"
+#pragma GCC diagnostic ignored "oops" /* { dg-error "oops" } */
+/* { dg-regexp {[^[:space:]]*: some warnings being treated as errors} } */
diff --git a/gcc/testsuite/g++.dg/pch/pr112319.Hs 
b/gcc/testsuite/g++.dg/pch/pr112319.Hs
new file mode 100644
index 000..3b6178bfae0
--- /dev/null
+++ b/gcc/testsuite/g++.dg/pch/pr112319.Hs
@@ -0,0 +1 @@
+/* This space intentionally left blank.  */
diff --git a/gcc/testsuite/gcc.dg/pch/pr112319.c 
b/gcc/testsuite/gcc.dg/pch/pr112319.c
new file mode 100644
index 000..043881463c5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pch/pr112319.c
@@ -0,0 +1,5 @@
+/* { dg-additional-options "-Wpragmas -save-temps" } */
+#include "pr112319.h"
+#pragma GCC diagnostic error "-Wpragmas"
+#pragma GCC diagnostic ignored "oops" /* { dg-error "oops" } */
+/* { dg-regexp {[^[:space:]]*: some warnings being treated as errors} } */
diff --git a/gcc/testsuite/gcc.dg/pch/pr112319.hs 
b/gcc/testsuite/gcc.dg/pch/pr112319.hs
new file mode 100644
index 000..3b6178bfae0
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pch/pr112319.hs
@@ -0,0 +1 @@
+/* This space intentionally left blank.  */

Re: [PATCH] libcpp: Improve the diagnostic for poisoned identifiers [PR36887]

[Lewis Hyatt]

On Thu, Oct 26, 2023 at 12:48 PM Christophe Lyon
 wrote:
>
> On Thu, 26 Oct 2023 at 18:18, Lewis Hyatt  wrote:
> >
> > On Thu, Oct 26, 2023 at 4:49 AM Christophe Lyon
> >  wrote:
> > > We have noticed that the new tests fail on aarch64 with:
> > > .../aarch64-unknown-linux-gnu/libc/usr/lib/crt1.o: in function `_start':
> > > .../sysdeps/aarch64/start.S:110:(.text+0x38): undefined reference to 
> > > `main'
> > >
> > > Looking at the test, I'd say it lacks a dg-do compile (to avoid
> > > linking), but how does it work on other targets?
> >
> > Thanks for pointing it out. I am definitely under the impression that
> > { dg-do compile } is the default and doesn't need to be specified, I
> > have never seen it not be the case before... Is that just not correct?
> > I tried it out on the cfarm (gcc185) for aarch64-redhat-linux and it
> > works for me there too, I tried the test individually and also as part
> > of the whole check-gcc-c++ target.
> >
> > I do see that there are target-dependent functions in
> > testsuite/lib/*.exp that will change dg-do-what-default under some
> > circumstances... but I also see in dg-pch.exp (which is the one
> > relevant for this test g++.dg/pch/pr36887.C) that dg-do-what-default
> > is set to compile explicitly.
>
> Indeed, thanks for checking.
>
> > Note sure what the best next step is, should I just add { dg-do
> > compile } since it's harmless in any case, or is there something else
> > worth looking into here? I'm not sure why I couldn't reproduce the
> > issue on the compile farm machine either, maybe you wouldn't mind
> > please check if adding this line fixes it for you anyway? Thanks...
>
> Can you share the compile line for this test in g++.log?
>

Sure, here is what I got on aarch64 for

make RUNTESTFLAGS=pch.exp=pr36887.C check-gcc-c++

For making the PCH:

xg++ -B/dev/shm/lhyatt/build/gcc/testsuite/g++/../../ ./pr36887.H
-fdiagnostics-plain-output -nostdinc++
-I/dev/shm/lhyatt/build/aarch64-unknown-linux-gnu/libstdc++-v3/include/aarch64-unknown-linux-gnu
-I/dev/shm/lhyatt/build/aarch64-unknown-linux-gnu/libstdc++-v3/include
-I/dev/shm/lhyatt/src/libstdc++-v3/libsupc++
-I/dev/shm/lhyatt/src/libstdc++-v3/include/backward
-I/dev/shm/lhyatt/src/libstdc++-v3/testsuite/util -fmessage-length=0
-g -o pr36887.H.gch

For compiling the test:

xg++ -B/dev/shm/lhyatt/build/gcc/testsuite/g++/../../
/dev/shm/lhyatt/src/gcc/testsuite/g++.dg/pch/pr36887.C
-fdiagnostics-plain-output -nostdinc++
-I/dev/shm/lhyatt/build/aarch64-unknown-linux-gnu/libstdc++-v3/include/aarch64-unknown-linux-gnu
-I/dev/shm/lhyatt/build/aarch64-unknown-linux-gnu/libstdc++-v3/include
-I/dev/shm/lhyatt/src/libstdc++-v3/libsupc++
-I/dev/shm/lhyatt/src/libstdc++-v3/include/backward
-I/dev/shm/lhyatt/src/libstdc++-v3/testsuite/util -fmessage-length=0
-g -I. -Dwith_PCH -S -o pr36887.s

(and then it repeats with -O2 added, or with -g removed as well)

> Actually I'm seeing several similar errors in our g++.log, not
> reported before because they were "pre-existing" failures.
> So something is confusing the testsuite and puts it into link mode.
>
> I am currently building from scratch, without our CI scripts to get
> some additional logs in a setup that probably matches yours. Then I
> should be able to add more traces a dejagnu level to understand what's
> happening.
>
> Thanks,
>
> Christophe

Re: [PATCH] libcpp: Improve the diagnostic for poisoned identifiers [PR36887]

[Lewis Hyatt]

On Thu, Oct 26, 2023 at 4:49 AM Christophe Lyon
 wrote:
> We have noticed that the new tests fail on aarch64 with:
> .../aarch64-unknown-linux-gnu/libc/usr/lib/crt1.o: in function `_start':
> .../sysdeps/aarch64/start.S:110:(.text+0x38): undefined reference to `main'
>
> Looking at the test, I'd say it lacks a dg-do compile (to avoid
> linking), but how does it work on other targets?

Thanks for pointing it out. I am definitely under the impression that
{ dg-do compile } is the default and doesn't need to be specified, I
have never seen it not be the case before... Is that just not correct?
I tried it out on the cfarm (gcc185) for aarch64-redhat-linux and it
works for me there too, I tried the test individually and also as part
of the whole check-gcc-c++ target.

I do see that there are target-dependent functions in
testsuite/lib/*.exp that will change dg-do-what-default under some
circumstances... but I also see in dg-pch.exp (which is the one
relevant for this test g++.dg/pch/pr36887.C) that dg-do-what-default
is set to compile explicitly.

Note sure what the best next step is, should I just add { dg-do
compile } since it's harmless in any case, or is there something else
worth looking into here? I'm not sure why I couldn't reproduce the
issue on the compile farm machine either, maybe you wouldn't mind
please check if adding this line fixes it for you anyway? Thanks...

-Lewis

ping: [PATCH] libcpp: Improve the diagnostic for poisoned identifiers [PR36887]

[Lewis Hyatt]

Hello-

May I please ping this one? Thanks...
https://gcc.gnu.org/pipermail/gcc-patches/2023-September/630967.html

-Lewis

On Wed, Sep 20, 2023 at 12:12 AM Lewis Hyatt  wrote:
>
> Hello-
>
> This patch implements the PR's request to add more information to the
> diagnostic issued for using a poisoned identifier. Bootstrapped + regtested
> all languages on x86-64 Linux. Does it look OK please? Thanks!
>
> -Lewis
>
> -- >8 --
>
> The PR requests an enhancement to the diagnostic issued for the use of a
> poisoned identifier. Currently, we show the location of the usage, but not
> the location which requested the poisoning, which would be helpful for the
> user if the decision to poison an identifier was made externally, such as
> in a library header.
>
> In order to output this information, we need to remember a location_t for
> each identifier that has been poisoned, and that data needs to be preserved
> as well in a PCH. One option would be to add a field to struct cpp_hashnode,
> but there is no convenient place to add it without increasing the size of
> the struct for all identifiers. Given this facility will be needed rarely,
> it seemed better to add a second hash map, which is handled PCH-wise the
> same as the current one in gcc/stringpool.cc. This hash map associates a new
> struct cpp_hashnode_extra with each identifier that needs one. Currently
> that struct only contains the new location_t, but it could be extended in
> the future if there is other ancillary data that may be convenient to put
> there for other purposes.
>
> libcpp/ChangeLog:
>
> PR preprocessor/36887
> * directives.cc (do_pragma_poison): Store in the extra hash map the
> location from which an identifier has been poisoned.
> * lex.cc (identifier_diagnostics_on_lex): When issuing a diagnostic
> for the use of a poisoned identifier, also add a note indicating the
> location from which it was poisoned.
> * identifiers.cc (alloc_node): Convert to template function.
> (_cpp_init_hashtable): Handle the new extra hash map.
> (_cpp_destroy_hashtable): Likewise.
> * include/cpplib.h (struct cpp_hashnode_extra): New struct.
> (cpp_create_reader): Update prototype to...
> * init.cc (cpp_create_reader): ...accept an argument for the extra
> hash table and pass it to _cpp_init_hashtable.
> * include/symtab.h (ht_lookup): New overload for convenience.
> * internal.h (struct cpp_reader): Add EXTRA_HASH_TABLE member.
> (_cpp_init_hashtable): Adjust prototype.
>
> gcc/c-family/ChangeLog:
>
> PR preprocessor/36887
> * c-opts.cc (c_common_init_options): Pass new extra hash map
> argument to cpp_create_reader().
>
> gcc/ChangeLog:
>
> PR preprocessor/36887
> * toplev.h (ident_hash_extra): Declare...
> * stringpool.cc (ident_hash_extra): ...this new global variable.
> (init_stringpool): Handle ident_hash_extra as well as ident_hash.
> (ggc_mark_stringpool): Likewise.
> (ggc_purge_stringpool): Likewise.
> (struct string_pool_data_extra): New struct.
> (spd2): New GC root variable.
> (gt_pch_save_stringpool): Use spd2 to handle ident_hash_extra,
> analogous to how spd is used to handle ident_hash.
> (gt_pch_restore_stringpool): Likewise.
>
> gcc/testsuite/ChangeLog:
>
> PR preprocessor/36887
> * c-c++-common/cpp/diagnostic-poison.c: New test.
> * g++.dg/pch/pr36887.C: New test.
> * g++.dg/pch/pr36887.Hs: New test.
> ---
>  libcpp/directives.cc  |  3 ++
>  libcpp/identifiers.cc | 42 +++--
>  libcpp/include/cpplib.h   | 21 ++---
>  libcpp/include/symtab.h   |  6 +++
>  libcpp/init.cc|  4 +-
>  libcpp/internal.h |  8 +++-
>  libcpp/lex.cc | 10 -
>  gcc/c-family/c-opts.cc|  2 +-
>  gcc/stringpool.cc | 45 +++
>  gcc/toplev.h  |  3 +-
>  .../c-c++-common/cpp/diagnostic-poison.c  | 13 ++
>  gcc/testsuite/g++.dg/pch/pr36887.C|  3 ++
>  gcc/testsuite/g++.dg/pch/pr36887.Hs   |  1 +
>  13 files changed, 134 insertions(+), 27 deletions(-)
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/diagnostic-poison.c
>  create mode 100644 gcc/testsuite/g++.dg/pch/pr36887.C
>  create mode 100644 gcc/testsuite/g++.dg/pch/pr36887.Hs
>
> diff --git a/libcpp/directives.cc b/libcpp/directives

Re: [PATCH] c++: Make -Wunknown-pragmas controllable by #pragma GCC diagnostic [PR89038]

[Lewis Hyatt]

On Thu, Oct 19, 2023 at 8:43 AM Marek Polacek  wrote:
>
> On Wed, Oct 18, 2023 at 05:15:42PM -0400, Lewis Hyatt wrote:
> > Hello-
> >
> > The PR points out that my fix for PR53431 was incomplete and did not handle
> > -Wunknown-pragmas. This is a one-line fix to correct that, is it OK for
> > trunk and for GCC 13 backport please? bootstrap + regtest all languages on
> > x86-64 Linux. Thanks!
>
> I think I can approve this, so, OK.  Thanks.
>

Great, thank you very much. Just to be safe, was that OK for the
backport as well?

-Lewis

[PATCH] c++: Make -Wunknown-pragmas controllable by #pragma GCC diagnostic [PR89038]

[Lewis Hyatt]

Hello-

The PR points out that my fix for PR53431 was incomplete and did not handle
-Wunknown-pragmas. This is a one-line fix to correct that, is it OK for
trunk and for GCC 13 backport please? bootstrap + regtest all languages on
x86-64 Linux. Thanks!

-Lewis

-- >8 --

As noted on the PR, commit r13-1544, the fix for PR53431, did not handle
the specific case of -Wunknown-pragmas, because that warning is issued
during preprocessing, but not by libcpp directly (it comes from the
cb_def_pragma callback).  Address that by handling this pragma in
addition to libcpp pragmas during the early pragma handler.

gcc/c-family/ChangeLog:

PR c++/89038
* c-pragma.cc (handle_pragma_diagnostic_impl):  Handle
-Wunknown-pragmas during early processing.

gcc/testsuite/ChangeLog:

PR c++/89038
* c-c++-common/cpp/Wunknown-pragmas-1.c: New test.
---
 gcc/c-family/c-pragma.cc|  3 ++-
 gcc/testsuite/c-c++-common/cpp/Wunknown-pragmas-1.c | 13 +
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/Wunknown-pragmas-1.c

diff --git a/gcc/c-family/c-pragma.cc b/gcc/c-family/c-pragma.cc
index 293311dd4ce..98dfb0f108b 100644
--- a/gcc/c-family/c-pragma.cc
+++ b/gcc/c-family/c-pragma.cc
@@ -963,7 +963,8 @@ handle_pragma_diagnostic_impl ()
   /* option_string + 1 to skip the initial '-' */
   unsigned int option_index = find_opt (data.option_str + 1, lang_mask);
 
-  if (early && !c_option_is_from_cpp_diagnostics (option_index))
+  if (early && !(c_option_is_from_cpp_diagnostics (option_index)
+|| option_index == OPT_Wunknown_pragmas))
 return;
 
   if (option_index == OPT_SPECIAL_unknown)
diff --git a/gcc/testsuite/c-c++-common/cpp/Wunknown-pragmas-1.c 
b/gcc/testsuite/c-c++-common/cpp/Wunknown-pragmas-1.c
new file mode 100644
index 000..fb58739e2bc
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/Wunknown-pragmas-1.c
@@ -0,0 +1,13 @@
+/* PR c++/89038 */
+/* { dg-additional-options "-Wunknown-pragmas" } */
+
+#pragma oops /* { dg-warning "-:-Wunknown-pragmas" } */
+#pragma GGC diagnostic push /* { dg-warning "-:-Wunknown-pragmas" } */
+#pragma GCC diagnostics push /* { dg-warning "-:-Wunknown-pragmas" } */
+
+/* Test we can disable the warnings.  */
+#pragma GCC diagnostic ignored "-Wunknown-pragmas"
+
+#pragma oops /* { dg-bogus "-:-Wunknown-pragmas" } */
+#pragma GGC diagnostic push /* { dg-bogus "-:-Wunknown-pragmas" } */
+#pragma GCC diagnostics push /* { dg-bogus "-:-Wunknown-pragmas" } */

Re: [PATCH] libcpp: testsuite: Add test for fixed _Pragma bug [PR82335]

[Lewis Hyatt]

May I please ping this one, and/or, is it something straightforward
enough I can just commit it as obvious? Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2023-October/631814.html

-Lewis

On Mon, Oct 2, 2023 at 6:23 PM Lewis Hyatt  wrote:
>
> Hello-
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82335 is another
> _Pragma-related bug that got fixed in GCC 12 but is still open. Before
> closing it out, I thought it would be good to add the testcase from that
> PR, which we don't have exactly in the testsuite already. Is it OK please?
> Thanks!
>
> -Lewis
>
> -- >8 --
>
> This PR was fixed by r12-4797 and r12-5454. Add test coverage from the PR
> that is not represented elsewhere.
>
> gcc/testsuite/ChangeLog:
>
> PR preprocessor/82335
> * c-c++-common/cpp/diagnostic-pragma-3.c: New test.
> ---
>  .../c-c++-common/cpp/diagnostic-pragma-3.c| 37 +++
>  1 file changed, 37 insertions(+)
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c
>
> diff --git a/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c 
> b/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c
> new file mode 100644
> index 000..459dcec73b3
> --- /dev/null
> +++ b/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c
> @@ -0,0 +1,37 @@
> +/* This is like diagnostic-pragma-2.c, but handles the case where everything
> +   is wrapped inside a macro, which previously caused additional issues 
> tracked
> +   in PR preprocessor/82335.  */
> +
> +/* { dg-do compile } */
> +/* { dg-additional-options "-save-temps -Wattributes -Wtype-limits" } */
> +
> +#define B _Pragma("GCC diagnostic push") \
> +  _Pragma("GCC diagnostic ignored \"-Wattributes\"")
> +#define E _Pragma("GCC diagnostic pop")
> +
> +#define X() B int __attribute((unknown_attr)) x; E
> +#define Y   B int __attribute((unknown_attr)) y; E
> +#define WRAP(x) x
> +
> +void test1(void)
> +{
> +  WRAP(X())
> +  WRAP(Y)
> +}
> +
> +/* Additional test provided on the PR.  */
> +#define PRAGMA(...) _Pragma(#__VA_ARGS__)
> +#define PUSH_IGN(X) PRAGMA(GCC diagnostic push) PRAGMA(GCC diagnostic 
> ignored X)
> +#define POP() PRAGMA(GCC diagnostic pop)
> +#define TEST(X, Y) \
> +  PUSH_IGN("-Wtype-limits") \
> +  int Y = (__typeof(X))-1 < 0; \
> +  POP()
> +
> +int test2()
> +{
> +  unsigned x;
> +  TEST(x, i1);
> +  WRAP(TEST(x, i2))
> +  return i1 + i2;
> +}

ping: [PATCH] preprocessor: c++: Support `#pragma GCC target' macros [PR87299]

[Lewis Hyatt]

On Tue, Sep 12, 2023 at 04:09:21PM -0400, Lewis Hyatt wrote:
> On Tue, Aug 8, 2023 at 5:53 PM Jason Merrill  wrote:
> >
> > On 7/31/23 22:22, Lewis Hyatt via Gcc-patches wrote:
> > > `#pragma GCC target' is not currently handled in preprocess-only mode 
> > > (e.g.,
> > > when running gcc -E or gcc -save-temps). As noted in the PR, this means 
> > > that
> > > if the target pragma defines any macros, those macros are not effective in
> > > preprocess-only mode. Similarly, such macros are not effective when
> > > compiling with C++ (even when compiling without -save-temps), because C++
> > > does not process the pragma until after all tokens have been obtained from
> > > libcpp, at which point it is too late for macro expansion to take place.
> > >
> > > Since r13-1544 and r14-2893, there is a general mechanism to handle 
> > > pragmas
> > > under these conditions as well, so resolve the PR by using the new "early
> > > pragma" support.
> > >
> > > toplev.cc required some changes because the target-specific handlers for
> > > `#pragma GCC target' may call target_reinit(), and toplev.cc was not 
> > > expecting
> > > that function to be called in preprocess-only mode.
> > >
> > > I added some additional testcases from the PR for x86. The other targets
> > > that support `#pragma GCC target' (aarch64, arm, nios2, powerpc, s390)
> > > already had tests verifying that the pragma sets macros as expected; here 
> > > I
> > > have added -save-temps to some of them, to test that it now works in
> > > preprocess-only mode as well.
> > >
> > > gcc/c-family/ChangeLog:
> > >
> > >   PR preprocessor/87299
> > >   * c-pragma.cc (init_pragma): Register `#pragma GCC target' and
> > >   related pragmas in preprocess-only mode, and enable early handling.
> > >   (c_reset_target_pragmas): New function refactoring code from...
> > >   (handle_pragma_reset_options): ...here.
> > >   * c-pragma.h (c_reset_target_pragmas): Declare.
> > >
> > > gcc/cp/ChangeLog:
> > >
> > >   PR preprocessor/87299
> > >   * parser.cc (cp_lexer_new_main): Call c_reset_target_pragmas ()
> > >   after preprocessing is complete, before starting compilation.
> > >
> > > gcc/ChangeLog:
> > >
> > >   PR preprocessor/87299
> > >   * toplev.cc (no_backend): New static global.
> > >   (finalize): Remove argument no_backend, which is now a
> > >   static global.
> > >   (process_options): Likewise.
> > >   (do_compile): Likewise.
> > >   (target_reinit): Don't do anything in preprocess-only mode.
> > >   (toplev::main): Adapt to no_backend change.
> > >   (toplev::finalize): Likewise.
> > >
> > > gcc/testsuite/ChangeLog:
> > >
> > >   PR preprocessor/87299
> > >   * c-c++-common/pragma-target-1.c: New test.
> > >   * c-c++-common/pragma-target-2.c: New test.
> > >   * g++.target/i386/pr87299-1.C: New test.
> > >   * g++.target/i386/pr87299-2.C: New test.
> > >   * gcc.target/i386/pr87299-1.c: New test.
> > >   * gcc.target/i386/pr87299-2.c: New test.
> > >   * gcc.target/s390/target-attribute/tattr-2.c: Add -save-temps to the
> > >   options, to test preprocess-only mode as well.
> > >   * gcc.target/aarch64/pragma_cpp_predefs_1.c: Likewise.
> > >   * gcc.target/arm/pragma_arch_attribute.c: Likewise.
> > >   * gcc.target/nios2/custom-fp-2.c: Likewise.
> > >   * gcc.target/powerpc/float128-3.c: Likewise.
> > > ---
> > >
> > > Notes:
> > >  Hello-
> > >
> > >  This patch fixes the PR by enabling early pragma handling for 
> > > `#pragma GCC
> > >  target' and related pragmas such as `#pragma GCC push_options'. I 
> > > did not
> > >  need to touch any target-specific code, however I did need to make a 
> > > change
> > >  to toplev.cc, affecting all targets, to make it safe to call 
> > > target_reinit()
> > >  in preprocess-only mode. (Otherwise, it would be necessary to modify 
> > > the
> > >  implementation of target pragmas in every target, to avoid this code 
> > > path.)
> > >  That was the only complication I ran into.
> > >
> > >  Regarding testin

[PATCH] libcpp: testsuite: Add test for fixed _Pragma bug [PR82335]

[Lewis Hyatt]

Hello-

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82335 is another
_Pragma-related bug that got fixed in GCC 12 but is still open. Before
closing it out, I thought it would be good to add the testcase from that
PR, which we don't have exactly in the testsuite already. Is it OK please?
Thanks!

-Lewis

-- >8 --

This PR was fixed by r12-4797 and r12-5454. Add test coverage from the PR
that is not represented elsewhere.

gcc/testsuite/ChangeLog:

PR preprocessor/82335
* c-c++-common/cpp/diagnostic-pragma-3.c: New test.
---
 .../c-c++-common/cpp/diagnostic-pragma-3.c| 37 +++
 1 file changed, 37 insertions(+)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c

diff --git a/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c 
b/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c
new file mode 100644
index 000..459dcec73b3
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/diagnostic-pragma-3.c
@@ -0,0 +1,37 @@
+/* This is like diagnostic-pragma-2.c, but handles the case where everything
+   is wrapped inside a macro, which previously caused additional issues tracked
+   in PR preprocessor/82335.  */
+
+/* { dg-do compile } */
+/* { dg-additional-options "-save-temps -Wattributes -Wtype-limits" } */
+
+#define B _Pragma("GCC diagnostic push") \
+  _Pragma("GCC diagnostic ignored \"-Wattributes\"")
+#define E _Pragma("GCC diagnostic pop")
+
+#define X() B int __attribute((unknown_attr)) x; E
+#define Y   B int __attribute((unknown_attr)) y; E
+#define WRAP(x) x
+
+void test1(void)
+{
+  WRAP(X())
+  WRAP(Y)
+}
+
+/* Additional test provided on the PR.  */
+#define PRAGMA(...) _Pragma(#__VA_ARGS__)
+#define PUSH_IGN(X) PRAGMA(GCC diagnostic push) PRAGMA(GCC diagnostic ignored 
X)
+#define POP() PRAGMA(GCC diagnostic pop)
+#define TEST(X, Y) \
+  PUSH_IGN("-Wtype-limits") \
+  int Y = (__typeof(X))-1 < 0; \
+  POP()
+
+int test2()
+{
+  unsigned x;
+  TEST(x, i1);
+  WRAP(TEST(x, i2))
+  return i1 + i2;
+}

[PATCH] libcpp: Improve the diagnostic for poisoned identifiers [PR36887]

[Lewis Hyatt]

Hello-

This patch implements the PR's request to add more information to the
diagnostic issued for using a poisoned identifier. Bootstrapped + regtested
all languages on x86-64 Linux. Does it look OK please? Thanks!

-Lewis

-- >8 --

The PR requests an enhancement to the diagnostic issued for the use of a
poisoned identifier. Currently, we show the location of the usage, but not
the location which requested the poisoning, which would be helpful for the
user if the decision to poison an identifier was made externally, such as
in a library header.

In order to output this information, we need to remember a location_t for
each identifier that has been poisoned, and that data needs to be preserved
as well in a PCH. One option would be to add a field to struct cpp_hashnode,
but there is no convenient place to add it without increasing the size of
the struct for all identifiers. Given this facility will be needed rarely,
it seemed better to add a second hash map, which is handled PCH-wise the
same as the current one in gcc/stringpool.cc. This hash map associates a new
struct cpp_hashnode_extra with each identifier that needs one. Currently
that struct only contains the new location_t, but it could be extended in
the future if there is other ancillary data that may be convenient to put
there for other purposes.

libcpp/ChangeLog:

PR preprocessor/36887
* directives.cc (do_pragma_poison): Store in the extra hash map the
location from which an identifier has been poisoned.
* lex.cc (identifier_diagnostics_on_lex): When issuing a diagnostic
for the use of a poisoned identifier, also add a note indicating the
location from which it was poisoned.
* identifiers.cc (alloc_node): Convert to template function.
(_cpp_init_hashtable): Handle the new extra hash map.
(_cpp_destroy_hashtable): Likewise.
* include/cpplib.h (struct cpp_hashnode_extra): New struct.
(cpp_create_reader): Update prototype to...
* init.cc (cpp_create_reader): ...accept an argument for the extra
hash table and pass it to _cpp_init_hashtable.
* include/symtab.h (ht_lookup): New overload for convenience.
* internal.h (struct cpp_reader): Add EXTRA_HASH_TABLE member.
(_cpp_init_hashtable): Adjust prototype.

gcc/c-family/ChangeLog:

PR preprocessor/36887
* c-opts.cc (c_common_init_options): Pass new extra hash map
argument to cpp_create_reader().

gcc/ChangeLog:

PR preprocessor/36887
* toplev.h (ident_hash_extra): Declare...
* stringpool.cc (ident_hash_extra): ...this new global variable.
(init_stringpool): Handle ident_hash_extra as well as ident_hash.
(ggc_mark_stringpool): Likewise.
(ggc_purge_stringpool): Likewise.
(struct string_pool_data_extra): New struct.
(spd2): New GC root variable.
(gt_pch_save_stringpool): Use spd2 to handle ident_hash_extra,
analogous to how spd is used to handle ident_hash.
(gt_pch_restore_stringpool): Likewise.

gcc/testsuite/ChangeLog:

PR preprocessor/36887
* c-c++-common/cpp/diagnostic-poison.c: New test.
* g++.dg/pch/pr36887.C: New test.
* g++.dg/pch/pr36887.Hs: New test.
---
 libcpp/directives.cc  |  3 ++
 libcpp/identifiers.cc | 42 +++--
 libcpp/include/cpplib.h   | 21 ++---
 libcpp/include/symtab.h   |  6 +++
 libcpp/init.cc|  4 +-
 libcpp/internal.h |  8 +++-
 libcpp/lex.cc | 10 -
 gcc/c-family/c-opts.cc|  2 +-
 gcc/stringpool.cc | 45 +++
 gcc/toplev.h  |  3 +-
 .../c-c++-common/cpp/diagnostic-poison.c  | 13 ++
 gcc/testsuite/g++.dg/pch/pr36887.C|  3 ++
 gcc/testsuite/g++.dg/pch/pr36887.Hs   |  1 +
 13 files changed, 134 insertions(+), 27 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/diagnostic-poison.c
 create mode 100644 gcc/testsuite/g++.dg/pch/pr36887.C
 create mode 100644 gcc/testsuite/g++.dg/pch/pr36887.Hs

diff --git a/libcpp/directives.cc b/libcpp/directives.cc
index ee5419d1f40..c5c938fda1d 100644
--- a/libcpp/directives.cc
+++ b/libcpp/directives.cc
@@ -1737,6 +1737,9 @@ do_pragma_poison (cpp_reader *pfile)
   NODE_NAME (hp));
   _cpp_free_definition (hp);
   hp->flags |= NODE_POISONED | NODE_DIAGNOSTIC;
+  const auto data = (cpp_hashnode_extra *)
+   ht_lookup (pfile->extra_hash_table, hp->ident, HT_ALLOC);
+  data->poisoned_loc = tok->src_loc;
 }
   pfile->state.poisoned_ok = 0;
 }
diff --git a/libcpp/identifiers.cc b/libcpp/identifiers.cc
index 7eccaa9bfd3..10cbbdf703d 100644
--- a/libcpp/identifiers.cc
+++ b/libcpp/identifiers.cc
@@ -27,24 +27,22 @@ alo

Re: [PATCH] libcpp: Fix ICE on #include after a line marker directive [PR61474]

[Lewis Hyatt]

On Tue, Sep 19, 2023 at 1:13 PM Marek Polacek  wrote:
>
> On Tue, Sep 19, 2023 at 06:08:50PM +0100, Richard Sandiford wrote:
> > Lewis Hyatt via Gcc-patches  writes:
> > > Hello-
> > >
> > > This fixes an old PR, bootstrap + regtest on x86-64 Linux. Please let me 
> > > know if it's ok? Thanks!
> > >
> > > -Lewis
> > >
> > > -- >8 --
> > >
> > > As noted in the PR, GCC will segfault if a file name is first seen in a
> > > linemarker directive, and then later seen in a normal #include.  This is
> > > because the fake include process adds the file to the cache with a null 
> > > PATH
> > > member. The normal #include finds this file in the cache and then attempts
> > > to use the null PATH.  Resolve by adding the file to the cache with a 
> > > unique
> > > starting directory, so that the fake entry will only be found by a
> > > subsequent fake include, not by a real one.
> > >
> > > libcpp/ChangeLog:
> > >
> > > PR preprocessor/61474
> > > * files.cc (_cpp_find_file): Set DONT_READ to TRUE for fake
> > > include files.
> > > (_cpp_fake_include): Pass a unique cpp_dir* address so
> > > the fake file will not be found when looked up for real.
> > >
> > > gcc/testsuite/ChangeLog:
> > >
> > > PR preprocessor/61474
> > > * c-c++-common/cpp/pr61474-2.h: New test.
> > > * c-c++-common/cpp/pr61474.c: New test.
> > > * c-c++-common/cpp/pr61474.h: New test.
> >
> > Neat fix!  I don't know this code very well, but I agree it looks
> > correct.  OK if no-one objects in 24 hours.
>
> Looks fine to me too, thanks Lewis.

Thank you both, much appreciated. I will push it tomorrow evening then.

-Lewis

[PATCH] libcpp: Fix ICE on #include after a line marker directive [PR61474]

[Lewis Hyatt via Gcc-patches]

Hello-

This fixes an old PR, bootstrap + regtest on x86-64 Linux. Please let me know 
if it's ok? Thanks!

-Lewis

-- >8 --

As noted in the PR, GCC will segfault if a file name is first seen in a
linemarker directive, and then later seen in a normal #include.  This is
because the fake include process adds the file to the cache with a null PATH
member. The normal #include finds this file in the cache and then attempts
to use the null PATH.  Resolve by adding the file to the cache with a unique
starting directory, so that the fake entry will only be found by a
subsequent fake include, not by a real one.

libcpp/ChangeLog:

PR preprocessor/61474
* files.cc (_cpp_find_file): Set DONT_READ to TRUE for fake
include files.
(_cpp_fake_include): Pass a unique cpp_dir* address so
the fake file will not be found when looked up for real.

gcc/testsuite/ChangeLog:

PR preprocessor/61474
* c-c++-common/cpp/pr61474-2.h: New test.
* c-c++-common/cpp/pr61474.c: New test.
* c-c++-common/cpp/pr61474.h: New test.
---
 libcpp/files.cc| 11 +--
 gcc/testsuite/c-c++-common/cpp/pr61474-2.h |  1 +
 gcc/testsuite/c-c++-common/cpp/pr61474.c   |  5 +
 gcc/testsuite/c-c++-common/cpp/pr61474.h   |  6 ++
 4 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pr61474-2.h
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pr61474.c
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pr61474.h

diff --git a/libcpp/files.cc b/libcpp/files.cc
index 43a8894b7de..27301d79fa4 100644
--- a/libcpp/files.cc
+++ b/libcpp/files.cc
@@ -541,7 +541,9 @@ _cpp_find_file (cpp_reader *pfile, const char *fname, 
cpp_dir *start_dir,
 = (kind == _cpp_FFK_PRE_INCLUDE
|| (pfile->buffer && pfile->buffer->file->implicit_preinclude));
 
-  if (kind != _cpp_FFK_FAKE)
+  if (kind == _cpp_FFK_FAKE)
+file->dont_read = true;
+  else
 /* Try each path in the include chain.  */
 for (;;)
   {
@@ -1490,7 +1492,12 @@ cpp_clear_file_cache (cpp_reader *pfile)
 void
 _cpp_fake_include (cpp_reader *pfile, const char *fname)
 {
-  _cpp_find_file (pfile, fname, pfile->buffer->file->dir, 0, _cpp_FFK_FAKE, 0);
+  /* It does not matter what are the contents of fake_source_dir, it will never
+ be inspected; we just use its address to uniquely signify that this file
+ was added as a fake include, so a later call to _cpp_find_file (to include
+ the file for real) won't find the fake one in the hash table.  */
+  static cpp_dir fake_source_dir;
+  _cpp_find_file (pfile, fname, &fake_source_dir, 0, _cpp_FFK_FAKE, 0);
 }
 
 /* Not everyone who wants to set system-header-ness on a buffer can
diff --git a/gcc/testsuite/c-c++-common/cpp/pr61474-2.h 
b/gcc/testsuite/c-c++-common/cpp/pr61474-2.h
new file mode 100644
index 000..6f70f09beec
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pr61474-2.h
@@ -0,0 +1 @@
+#pragma once
diff --git a/gcc/testsuite/c-c++-common/cpp/pr61474.c 
b/gcc/testsuite/c-c++-common/cpp/pr61474.c
new file mode 100644
index 000..f835a40fc7a
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pr61474.c
@@ -0,0 +1,5 @@
+/* { dg-do preprocess } */
+#include "pr61474.h"
+/* Make sure that the file can be included for real, after it was
+   fake-included by the linemarker directives in pr61474.h.  */
+#include "pr61474-2.h"
diff --git a/gcc/testsuite/c-c++-common/cpp/pr61474.h 
b/gcc/testsuite/c-c++-common/cpp/pr61474.h
new file mode 100644
index 000..d9e8c3a1fec
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pr61474.h
@@ -0,0 +1,6 @@
+/* Create a fake include for pr61474-2.h and exercise looking it up.  */
+/* Use #pragma once to check also that the fake-include entry in the file
+   cache does not cause a problem in libcpp/files.cc:has_unique_contents().  */
+#pragma once
+# 1 "pr61474-2.h" 1
+# 2 "pr61474-2.h" 1

Re: [PATCH] preprocessor: c++: Support `#pragma GCC target' macros [PR87299]

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 8, 2023 at 5:53 PM Jason Merrill  wrote:
>
> On 7/31/23 22:22, Lewis Hyatt via Gcc-patches wrote:
> > `#pragma GCC target' is not currently handled in preprocess-only mode (e.g.,
> > when running gcc -E or gcc -save-temps). As noted in the PR, this means that
> > if the target pragma defines any macros, those macros are not effective in
> > preprocess-only mode. Similarly, such macros are not effective when
> > compiling with C++ (even when compiling without -save-temps), because C++
> > does not process the pragma until after all tokens have been obtained from
> > libcpp, at which point it is too late for macro expansion to take place.
> >
> > Since r13-1544 and r14-2893, there is a general mechanism to handle pragmas
> > under these conditions as well, so resolve the PR by using the new "early
> > pragma" support.
> >
> > toplev.cc required some changes because the target-specific handlers for
> > `#pragma GCC target' may call target_reinit(), and toplev.cc was not 
> > expecting
> > that function to be called in preprocess-only mode.
> >
> > I added some additional testcases from the PR for x86. The other targets
> > that support `#pragma GCC target' (aarch64, arm, nios2, powerpc, s390)
> > already had tests verifying that the pragma sets macros as expected; here I
> > have added -save-temps to some of them, to test that it now works in
> > preprocess-only mode as well.
> >
> > gcc/c-family/ChangeLog:
> >
> >   PR preprocessor/87299
> >   * c-pragma.cc (init_pragma): Register `#pragma GCC target' and
> >   related pragmas in preprocess-only mode, and enable early handling.
> >   (c_reset_target_pragmas): New function refactoring code from...
> >   (handle_pragma_reset_options): ...here.
> >   * c-pragma.h (c_reset_target_pragmas): Declare.
> >
> > gcc/cp/ChangeLog:
> >
> >   PR preprocessor/87299
> >   * parser.cc (cp_lexer_new_main): Call c_reset_target_pragmas ()
> >   after preprocessing is complete, before starting compilation.
> >
> > gcc/ChangeLog:
> >
> >   PR preprocessor/87299
> >   * toplev.cc (no_backend): New static global.
> >   (finalize): Remove argument no_backend, which is now a
> >   static global.
> >   (process_options): Likewise.
> >   (do_compile): Likewise.
> >   (target_reinit): Don't do anything in preprocess-only mode.
> >   (toplev::main): Adapt to no_backend change.
> >   (toplev::finalize): Likewise.
> >
> > gcc/testsuite/ChangeLog:
> >
> >   PR preprocessor/87299
> >   * c-c++-common/pragma-target-1.c: New test.
> >   * c-c++-common/pragma-target-2.c: New test.
> >   * g++.target/i386/pr87299-1.C: New test.
> >   * g++.target/i386/pr87299-2.C: New test.
> >   * gcc.target/i386/pr87299-1.c: New test.
> >   * gcc.target/i386/pr87299-2.c: New test.
> >   * gcc.target/s390/target-attribute/tattr-2.c: Add -save-temps to the
> >   options, to test preprocess-only mode as well.
> >   * gcc.target/aarch64/pragma_cpp_predefs_1.c: Likewise.
> >   * gcc.target/arm/pragma_arch_attribute.c: Likewise.
> >   * gcc.target/nios2/custom-fp-2.c: Likewise.
> >   * gcc.target/powerpc/float128-3.c: Likewise.
> > ---
> >
> > Notes:
> >  Hello-
> >
> >  This patch fixes the PR by enabling early pragma handling for `#pragma 
> > GCC
> >  target' and related pragmas such as `#pragma GCC push_options'. I did 
> > not
> >  need to touch any target-specific code, however I did need to make a 
> > change
> >  to toplev.cc, affecting all targets, to make it safe to call 
> > target_reinit()
> >  in preprocess-only mode. (Otherwise, it would be necessary to modify 
> > the
> >  implementation of target pragmas in every target, to avoid this code 
> > path.)
> >  That was the only complication I ran into.
> >
> >  Regarding testing, I did: (thanks to GCC compile farm for the non-x86
> >  targets)
> >
> >  bootstrap + regtest all languages - x86_64-pc-linux-gnu
> >  bootstrap + regtest c/c++ - powerpc64le-unknown-linux-gnu,
> >  aarch64-unknown-linux-gnu
> >
> >  The following backends also implement this pragma so ought to be 
> > tested:
> >  arm
> >  nios2
> >  s390
> >
> >  I am not able to test those directly. I did add coverage to

Ping: [PATCH] testsuite: Add test for already-fixed issue with _Pragma expansion [PR90400]

[Lewis Hyatt via Gcc-patches]

Hello-

May I please ping this one? It's adding a testcase prior to closing
the PR. Thanks!
https://gcc.gnu.org/pipermail/gcc-patches/2023-August/628488.html

-Lewis

On Fri, Aug 25, 2023 at 4:46 PM Lewis Hyatt  wrote:
>
> Hello-
>
> This is adding a testcase for a PR that was already incidentally fixed. OK
> to commit please? Thanks...
>
> -Lewis
>
> -- >8 --
>
> The PR was fixed by r12-5454. Since the fix was somewhat incidental,
> although related, add a testcase from PR90400 too before closing it out.
>
> gcc/testsuite/ChangeLog:
>
> PR preprocessor/90400
> * c-c++-common/cpp/pr90400.c: New test.
> ---
>  gcc/testsuite/c-c++-common/cpp/pr90400.c | 14 ++
>  1 file changed, 14 insertions(+)
>  create mode 100644 gcc/testsuite/c-c++-common/cpp/pr90400.c
>
> diff --git a/gcc/testsuite/c-c++-common/cpp/pr90400.c 
> b/gcc/testsuite/c-c++-common/cpp/pr90400.c
> new file mode 100644
> index 000..4f2cab8d6ab
> --- /dev/null
> +++ b/gcc/testsuite/c-c++-common/cpp/pr90400.c
> @@ -0,0 +1,14 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-save-temps" } */
> +/* PR preprocessor/90400 */
> +
> +#define OUTER(x) x
> +#define FOR(x) _Pragma ("GCC unroll 0") for (x)
> +void f ()
> +{
> +/* If the pragma were to be seen prior to the expansion of FOR, as was
> +   the case before r12-5454, then the unroll pragma would complain
> +   because the immediately following statement would be ";" rather than
> +   a loop.  */
> +OUTER (; FOR (int i = 0; i != 1; ++i);) /* { dg-bogus {statement 
> expected before ';' token} } */
> +}

[PATCH] testsuite: Add test for already-fixed issue with _Pragma expansion [PR90400]

[Lewis Hyatt via Gcc-patches]

Hello-

This is adding a testcase for a PR that was already incidentally fixed. OK
to commit please? Thanks...

-Lewis

-- >8 --

The PR was fixed by r12-5454. Since the fix was somewhat incidental,
although related, add a testcase from PR90400 too before closing it out.

gcc/testsuite/ChangeLog:

PR preprocessor/90400
* c-c++-common/cpp/pr90400.c: New test.
---
 gcc/testsuite/c-c++-common/cpp/pr90400.c | 14 ++
 1 file changed, 14 insertions(+)
 create mode 100644 gcc/testsuite/c-c++-common/cpp/pr90400.c

diff --git a/gcc/testsuite/c-c++-common/cpp/pr90400.c 
b/gcc/testsuite/c-c++-common/cpp/pr90400.c
new file mode 100644
index 000..4f2cab8d6ab
--- /dev/null
+++ b/gcc/testsuite/c-c++-common/cpp/pr90400.c
@@ -0,0 +1,14 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-save-temps" } */
+/* PR preprocessor/90400 */
+
+#define OUTER(x) x
+#define FOR(x) _Pragma ("GCC unroll 0") for (x)
+void f ()
+{
+/* If the pragma were to be seen prior to the expansion of FOR, as was
+   the case before r12-5454, then the unroll pragma would complain
+   because the immediately following statement would be ";" rather than
+   a loop.  */
+OUTER (; FOR (int i = 0; i != 1; ++i);) /* { dg-bogus {statement expected 
before ';' token} } */
+}

Re: [PATCH v4 3/8] diagnostics: Refactor class file_cache_slot

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 03:39:40PM -0400, David Malcolm wrote:
> On Tue, 2023-08-15 at 13:58 -0400, Lewis Hyatt wrote:
> > On Tue, Aug 15, 2023 at 11:43:05AM -0400, David Malcolm wrote:
> > > On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > > > Class file_cache_slot in input.cc is used to query specific lines
> > > > of source
> > > > code from a file when needed by diagnostics infrastructure. This
> > > > will be
> > > > extended in a subsequent patch to support obtaining the source
> > > > code from
> > > > in-memory generated buffers rather than from a file. The present
> > > > patch
> > > > refactors class file_cache_slot, putting most of the logic into a
> > > > new base
> > > > class cache_data_source, in preparation for reusing that code in
> > > > the next
> > > > patch. There is no change in functionality yet.
> > > > 
> 
> [...snip...]
> 
> > > 
> > > I confess I had to reread both this and patch 4/8 to make sense of
> > > this; this is probably one of those cases where it's harder to read
> > > in
> > > patch form than as source, but I think I now understand the new
> > > implementation.
> > 
> > Yes, sorry about that. I hope at least splitting into two patches
> > here made it
> > a little easier.
> > 
> > > 
> > > Did you try testing this with valgrind (e.g. "make selftest-
> > > valgrind")?
> > > 
> > 
> > Oh interesting, was not aware of this. I think it shows that new
> > leaks were
> > not introduced with the patch series.
> > 
> 
> [...snip...]
> 
> > 
> > 
> > > I don't think we have any selftest coverage for "\r" in the line-
> > > break
> > > handling; that would be good to add.
> > > 
> > > This patch is OK for trunk once the rest of the kit is approved.
> > 
> > Thank you. To be clear, were you suggesting to add selftest coverage
> > for \r
> > endings now, or in a follow up?
> 
> The former, please, so that we can sure that the patch doesn't
> introduce any buffer overreads etc.
> 
> Thanks
> Dave
>

The following (incremental to patch 5/8 or after) adds selftest coverage for
alternate line endings. I hope things aren't too unclear this way; I can
resend updated versions of some or all of the patches from scratch, if useful.

AFAIK this is the current status of things:

Patch 1/8: Reviewed, updated version incorporating feedback has not been acked
yet, at: https://gcc.gnu.org/pipermail/gcc-patches/2023-August/627250.html

Patch 2/8: OKed, pending tweak to reject fixit hints in generated data, which
was sent incrementally here:
https://gcc.gnu.org/pipermail/gcc-patches/2023-August/627405.html

Patch 3/8: OKed, pending new selftest attached to this email.

Patch 4/8: OKed, pending tweak to assert on non-NULL buffers which was sent
incrementally here:
https://gcc.gnu.org/pipermail/gcc-patches/2023-August/628283.html

Patch 5/8: OKed

Patch 6/8: OKed

Patch 7/8: Not reviewed yet

Patch 8/8: Waiting additional feedback from you, perhaps SARIF need not worry
about this for now and should just ignore generated data locations.

Thanks again for taking the time to go through this, I hope it will prove
worth it.

-Lewis

-- >8 --

gcc/ChangeLog:

* input.cc (test_reading_source_line): Test additional cases,
including generated data and alternate line endings.
(input_cc_tests): Adapt to test_reading_source_line() changes.

diff --git a/gcc/input.cc b/gcc/input.cc
index 4c99df7a205..72274732c6c 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -2392,30 +2392,51 @@ test_make_location_nonpure_range_endpoints (const 
line_table_case &case_)
 /* Verify reading of input files (e.g. for caret-based diagnostics).  */
 
 static void
-test_reading_source_line ()
+test_reading_source_line (bool generated, const char *e1, const char *e2)
 {
   /* Create a tempfile and write some text to it.  */
+  const char *line1 = "01234567890123456789";
+  const char *line2 = "This is the test text";
+  const char *line3 = "This is the 3rd line";
+  char content[72];
+  const int content_len = snprintf (content, sizeof (content),
+   "%s%s%s%s%s",
+   line1, e1, line2, e2, line3);
+  ASSERT_LT (content_len, (int)sizeof (content));
   temp_source_file tmp (SELFTEST_LOCATION, ".txt",
-   "01234567890123456789\n"
-   "This is the test text\n"
-   "This is the 3rd line");
+   c

Re: [PATCH v4 4/8] diagnostics: Support obtaining source code lines from generated data buffers

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 04:08:47PM -0400, Lewis Hyatt wrote:
> On Tue, Aug 15, 2023 at 3:46 PM David Malcolm  wrote:
> >
> > On Tue, 2023-08-15 at 14:15 -0400, Lewis Hyatt wrote:
> > > On Tue, Aug 15, 2023 at 12:15:15PM -0400, David Malcolm wrote:
> > > > On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > > > > This patch enhances location_get_source_line(), which is the
> > > > > primary
> > > > > interface provided by the diagnostics infrastructure to obtain
> > > > > the line of
> > > > > source code corresponding to a given location, so that it
> > > > > understands
> > > > > generated data locations in addition to normal file-based
> > > > > locations. This
> > > > > involves changing the argument to location_get_source_line() from
> > > > > a plain
> > > > > file name, to a source_id object that can represent either type
> > > > > of location.
> > > > >
> >
> > [...]
> >
> > > > >
> > > > >
> > > > > diff --git a/gcc/input.cc b/gcc/input.cc
> > > > > index 9377020b460..790279d4273 100644
> > > > > --- a/gcc/input.cc
> > > > > +++ b/gcc/input.cc
> > > > > @@ -207,6 +207,28 @@ private:
> > > > >void maybe_grow ();
> > > > >  };
> > > > >
> > > > > +/* This is the implementation of cache_data_source for generated
> > > > > +   data that is already in memory.  */
> > > > > +class data_cache_slot final : public cache_data_source
> > > >
> > > > It occurred to me: why are we caching accessing a buffer that's
> > > > already
> > > > in memory - but we're also caching the line-splitting information,
> > > > and
> > > > providing the line-splitting algorithm with a consistent interface
> > > > to
> > > > the data, right?
> > > >
> > >
> > > Yeah, for the current _Pragma use case, multi-line buffers are not
> > > going to
> > > be common, but they can occur. I was mainly motivated by the
> > > consistent
> > > interface, and by the assumption that the overhead is not critical
> > > given a
> > > diagnostic is being issued.
> >
> > (nods)
> >
> > >
> > > > [...snip...]
> > > >
> > > > > @@ -397,6 +434,15 @@ diagnostics_file_cache_forcibly_evict_file
> > > > > (const char *file_path)
> > > > >global_dc->m_file_cache->forcibly_evict_file (file_path);
> > > > >  }
> > > > >
> > > > > +void
> > > > > +diagnostics_file_cache_forcibly_evict_data (const char *data,
> > > > > +   unsigned int
> > > > > data_len)
> > > > > +{
> > > > > +  if (!global_dc->m_file_cache)
> > > > > +return;
> > > > > +  global_dc->m_file_cache->forcibly_evict_data (data, data_len);
> > > >
> > > > Maybe we should rename diagnostic_context's m_file_cache to
> > > > m_source_cache?  (and class file_cache for that matter?)  But if
> > > > so,
> > > > that can/should be a followup/separate patch.
> > > >
> > >
> > > Yes, we should. Believe it or not, I was trying to minimize the size
> > > of the
> > > patch :)
> >
> > :)
> >
> > Thanks for splitting it up, BTW.
> >
> > [...]
> >
> >
> > > >
> > > > > @@ -912,26 +1000,22 @@ cache_data_source::read_line_num (size_t
> > > > > line_num,
> > > > > If the function fails, a NULL char_span is returned.  */
> > > > >
> > > > >  char_span
> > > > > -location_get_source_line (const char *file_path, int line)
> > > > > +location_get_source_line (source_id src, int line)
> > > > >  {
> > > > > -  const char *buffer = NULL;
> > > > > -  ssize_t len;
> > > > > -
> > > > > -  if (line == 0)
> > > > > -return char_span (NULL, 0);
> > > > > -
> > > > > -  if (file_path == NULL)
> > > > > -return char_span (NULL, 0);
> > > > > +  const char_span fail (nullptr, 0);
> > > > > +  if (!src || line <= 0)
> > > &g

Re: [PATCH v4 4/8] diagnostics: Support obtaining source code lines from generated data buffers

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 3:46 PM David Malcolm  wrote:
>
> On Tue, 2023-08-15 at 14:15 -0400, Lewis Hyatt wrote:
> > On Tue, Aug 15, 2023 at 12:15:15PM -0400, David Malcolm wrote:
> > > On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > > > This patch enhances location_get_source_line(), which is the
> > > > primary
> > > > interface provided by the diagnostics infrastructure to obtain
> > > > the line of
> > > > source code corresponding to a given location, so that it
> > > > understands
> > > > generated data locations in addition to normal file-based
> > > > locations. This
> > > > involves changing the argument to location_get_source_line() from
> > > > a plain
> > > > file name, to a source_id object that can represent either type
> > > > of location.
> > > >
>
> [...]
>
> > > >
> > > >
> > > > diff --git a/gcc/input.cc b/gcc/input.cc
> > > > index 9377020b460..790279d4273 100644
> > > > --- a/gcc/input.cc
> > > > +++ b/gcc/input.cc
> > > > @@ -207,6 +207,28 @@ private:
> > > >void maybe_grow ();
> > > >  };
> > > >
> > > > +/* This is the implementation of cache_data_source for generated
> > > > +   data that is already in memory.  */
> > > > +class data_cache_slot final : public cache_data_source
> > >
> > > It occurred to me: why are we caching accessing a buffer that's
> > > already
> > > in memory - but we're also caching the line-splitting information,
> > > and
> > > providing the line-splitting algorithm with a consistent interface
> > > to
> > > the data, right?
> > >
> >
> > Yeah, for the current _Pragma use case, multi-line buffers are not
> > going to
> > be common, but they can occur. I was mainly motivated by the
> > consistent
> > interface, and by the assumption that the overhead is not critical
> > given a
> > diagnostic is being issued.
>
> (nods)
>
> >
> > > [...snip...]
> > >
> > > > @@ -397,6 +434,15 @@ diagnostics_file_cache_forcibly_evict_file
> > > > (const char *file_path)
> > > >global_dc->m_file_cache->forcibly_evict_file (file_path);
> > > >  }
> > > >
> > > > +void
> > > > +diagnostics_file_cache_forcibly_evict_data (const char *data,
> > > > +   unsigned int
> > > > data_len)
> > > > +{
> > > > +  if (!global_dc->m_file_cache)
> > > > +return;
> > > > +  global_dc->m_file_cache->forcibly_evict_data (data, data_len);
> > >
> > > Maybe we should rename diagnostic_context's m_file_cache to
> > > m_source_cache?  (and class file_cache for that matter?)  But if
> > > so,
> > > that can/should be a followup/separate patch.
> > >
> >
> > Yes, we should. Believe it or not, I was trying to minimize the size
> > of the
> > patch :)
>
> :)
>
> Thanks for splitting it up, BTW.
>
> [...]
>
>
> > >
> > > > @@ -912,26 +1000,22 @@ cache_data_source::read_line_num (size_t
> > > > line_num,
> > > > If the function fails, a NULL char_span is returned.  */
> > > >
> > > >  char_span
> > > > -location_get_source_line (const char *file_path, int line)
> > > > +location_get_source_line (source_id src, int line)
> > > >  {
> > > > -  const char *buffer = NULL;
> > > > -  ssize_t len;
> > > > -
> > > > -  if (line == 0)
> > > > -return char_span (NULL, 0);
> > > > -
> > > > -  if (file_path == NULL)
> > > > -return char_span (NULL, 0);
> > > > +  const char_span fail (nullptr, 0);
> > > > +  if (!src || line <= 0)
> > > > +return fail;
> > >
> > > Looking at source_id's operator bool, are there effectively three
> > > kinds
> > > of source_id?
> > >
> > > (a) file names
> > > (b) generated buffer
> > > (c) NULL == m_filename_or_buffer
> > >
> > > What does (c) mean?  Is it a "something's gone wrong/error" state?
> > > Or
> > > is this more a special-case of (a)? (in that the m_len for such a
> > > case
> > > would be zero)
> > >
> > > Should sou

Re: [PATCH v4 4/8] diagnostics: Support obtaining source code lines from generated data buffers

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 12:15:15PM -0400, David Malcolm wrote:
> On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > This patch enhances location_get_source_line(), which is the primary
> > interface provided by the diagnostics infrastructure to obtain the line of
> > source code corresponding to a given location, so that it understands
> > generated data locations in addition to normal file-based locations. This
> > involves changing the argument to location_get_source_line() from a plain
> > file name, to a source_id object that can represent either type of location.
> > 
> > gcc/ChangeLog:
> > 
> > * input.cc (class data_cache_slot): New class.
> > (file_cache::lookup_data): New function.
> > (diagnostics_file_cache_forcibly_evict_data): New function.
> > (file_cache::forcibly_evict_data): New function.
> > (file_cache::evicted_cache_tab_entry): Generalize (via a template)
> > to work for both file_cache_slot and data_cache_slot.
> > (file_cache::add_file): Adapt for new interface to
> > evicted_cache_tab_entry.
> > (file_cache::add_data): New function.
> > (data_cache_slot::create): New function.
> > (file_cache::file_cache): Support the new m_data_slots member.
> > (file_cache::~file_cache): Likewise.
> > (file_cache::lookup_or_add_data): New function.
> > (file_cache::lookup_or_add): New function that calls either
> > lookup_or_add_data or lookup_or_add_file as appropriate.
> > (location_get_source_line): Change the FILE_PATH argument to a
> > source_id SRC, and use it to support obtaining source lines from
> > generated data as well as from files.
> > (location_compute_display_column): Support generated data using the
> > new features of location_get_source_line.
> > (dump_location_info): Likewise.
> > * input.h (location_get_source_line): Adjust prototype. Add a new
> > convenience overload taking an expanded_location.
> > (class cache_data_source): Declare.
> > (class data_cache_slot): Declare.
> > (class file_cache): Declare new members.
> > (diagnostics_file_cache_forcibly_evict_data): Declare.
> > ---
> >  gcc/input.cc | 171 ---
> >  gcc/input.h  |  23 +--
> >  2 files changed, 153 insertions(+), 41 deletions(-)
> > 
> > diff --git a/gcc/input.cc b/gcc/input.cc
> > index 9377020b460..790279d4273 100644
> > --- a/gcc/input.cc
> > +++ b/gcc/input.cc
> > @@ -207,6 +207,28 @@ private:
> >void maybe_grow ();
> >  };
> >  
> > +/* This is the implementation of cache_data_source for generated
> > +   data that is already in memory.  */
> > +class data_cache_slot final : public cache_data_source
> 
> It occurred to me: why are we caching accessing a buffer that's already
> in memory - but we're also caching the line-splitting information, and
> providing the line-splitting algorithm with a consistent interface to
> the data, right?
>

Yeah, for the current _Pragma use case, multi-line buffers are not going to
be common, but they can occur. I was mainly motivated by the consistent
interface, and by the assumption that the overhead is not critical given a
diagnostic is being issued.

> [...snip...]
> 
> > @@ -397,6 +434,15 @@ diagnostics_file_cache_forcibly_evict_file (const char 
> > *file_path)
> >global_dc->m_file_cache->forcibly_evict_file (file_path);
> >  }
> >  
> > +void
> > +diagnostics_file_cache_forcibly_evict_data (const char *data,
> > +   unsigned int data_len)
> > +{
> > +  if (!global_dc->m_file_cache)
> > +return;
> > +  global_dc->m_file_cache->forcibly_evict_data (data, data_len);
> 
> Maybe we should rename diagnostic_context's m_file_cache to
> m_source_cache?  (and class file_cache for that matter?)  But if so,
> that can/should be a followup/separate patch.
>

Yes, we should. Believe it or not, I was trying to minimize the size of the
patch :) So I didn't make such changes, but they will make things more
clear.

> [...snip...]
>  
> > @@ -525,10 +582,22 @@ file_cache_slot::create (const 
> > file_cache::input_context &in_context,
> >return true;
> >  }
> >  
> > +void
> > +data_cache_slot::create (const char *data, unsigned int data_len,
> > +unsigned int highest_use_count)
> > +{
> > +  reset ();
> > +  o

Re: [PATCH v4 3/8] diagnostics: Refactor class file_cache_slot

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 11:43:05AM -0400, David Malcolm wrote:
> On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > Class file_cache_slot in input.cc is used to query specific lines of source
> > code from a file when needed by diagnostics infrastructure. This will be
> > extended in a subsequent patch to support obtaining the source code from
> > in-memory generated buffers rather than from a file. The present patch
> > refactors class file_cache_slot, putting most of the logic into a new base
> > class cache_data_source, in preparation for reusing that code in the next
> > patch. There is no change in functionality yet.
> > 
> > gcc/ChangeLog:
> > 
> > * input.cc (class file_cache_slot): Refactor functionality into a
> > new base class...
> > (class cache_data_source): ...here.
> > (file_cache::forcibly_evict_file): Adapt for refactoring.
> > (file_cache_slot::evict): Renamed to...
> > (file_cache_slot::reset): ...this, and partially refactored into
> > base class...
> > (cache_data_source::reset): ...here.
> > (file_cache_slot::get_full_file_content): Moved into base class...
> > (cache_data_source::get_full_file_content): ...here.
> > (file_cache_slot::create): Adapt for refactoring.
> > (file_cache_slot::file_cache_slot): Refactor partially into...
> > (cache_data_source::cache_data_source): ...here.
> > (file_cache_slot::~file_cache_slot): Refactor partially into...
> > (cache_data_source::~cache_data_source): ...here.
> > (file_cache_slot::needs_read_p): Remove.
> > (file_cache_slot::needs_grow_p): Remove.
> > (file_cache_slot::maybe_grow): Adapt for refactoring.
> > (file_cache_slot::read_data): Refactored, along with...
> > (file_cache_slot::maybe_read_data): this, into...
> > (file_cache_slot::get_more_data): ...here.
> > (find_end_of_line): Change interface to take a pair of pointers,
> > rather than a pointer + length.
> > (file_cache_slot::get_next_line): Refactored into...
> > (cache_data_source::get_next_line): ...here.
> > (file_cache_slot::goto_next_line): Refactored into...
> > (cache_data_source::goto_next_line): ...here.
> > (file_cache_slot::read_line_num): Refactored into...
> > (cache_data_source::read_line_num): ...here.
> > (location_get_source_line): Fix const-correctness as necessitated by
> > new interface.
> > ---
> >  gcc/input.cc | 513 +++
> >  1 file changed, 235 insertions(+), 278 deletions(-)
> > 
> 
> I confess I had to reread both this and patch 4/8 to make sense of
> this; this is probably one of those cases where it's harder to read in
> patch form than as source, but I think I now understand the new
> implementation.

Yes, sorry about that. I hope at least splitting into two patches here made it
a little easier.

> 
> Did you try testing this with valgrind (e.g. "make selftest-valgrind")?
>

Oh interesting, was not aware of this. I think it shows that new leaks were
not introduced with the patch series.

BEFORE patch series:
==1572278==
-fself-test: 7634593 pass(es) in 22.799240 seconds
==1572278==
==1572278== HEAP SUMMARY:
==1572278== in use at exit: 1,083,255 bytes in 2,394 blocks
==1572278==   total heap usage: 2,704,869 allocs, 2,702,475 frees, 
1,257,334,536 bytes allocated
==1572278==
==1572278== 8,032 bytes in 1 blocks are possibly lost in loss record 639 of 657
==1572278==at 0x4848899: malloc (in 
/usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==1572278==by 0x21FE1CB: xmalloc (xmalloc.c:149)
==1572278==by 0x21B02E0: new_buff (lex.cc:4767)
==1572278==by 0x21B02E0: _cpp_get_buff (lex.cc:4800)
==1572278==by 0x21ACC80: cpp_create_reader(c_lang, ht*, line_maps*) 
(init.cc:289)
==1572278==by 0xA64282: c_common_init_options(unsigned int, 
cl_decoded_option*) (c-opts.cc:237)
==1572278==by 0x95E479: toplev::main(int, char**) (toplev.cc:2241)
==1572278==by 0x960B2D: main (main.cc:39)
==1572278==
==1572278== LEAK SUMMARY:
==1572278==definitely lost: 0 bytes in 0 blocks
==1572278==indirectly lost: 0 bytes in 0 blocks
==1572278==  possibly lost: 8,032 bytes in 1 blocks
==1572278==still reachable: 1,075,223 bytes in 2,393 blocks
==1572278== suppressed: 0 bytes in 0 blocks
==1572278== Reachable blocks (those to which a pointer was found) are not shown.
==1572278== To see them, rerun with: --leak-check=full --show-leak-kinds=all
==1572278==
==1572278== For lists of detected and suppressed errors, rerun

Re: [PATCH v4 8/8] diagnostics: Support generated data locations in SARIF output

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 15, 2023 at 01:04:04PM -0400, David Malcolm wrote:
> On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > The diagnostics routines for SARIF output need to read the source code back
> > in, so that they can generate "snippet" and "content" records, so they need 
> > to
> > be able to cope with generated data locations.  Add support for that in
> > diagnostic-format-sarif.cc.
> > 
> > gcc/ChangeLog:
> > 
> > * diagnostic-format-sarif.cc (class sarif_builder): Adapt interface
> > to support generated data locations.
> > (sarif_builder::maybe_make_physical_location_object): Change the
> > m_filenames hash_set to support generated data.
> > (sarif_builder::make_artifact_location_object): Use a source_id 
> > rather
> > than a plain file name.
> > (sarif_builder::maybe_make_region_object): Adapt to
> > expanded_location interface changes.
> > (sarif_builder::maybe_make_region_object_for_context): Likewise.
> > (sarif_builder::make_artifact_object): Likewise.
> > (sarif_builder::make_run_object): Handle generated data.
> > (sarif_builder::maybe_make_artifact_content_object): Likewise.
> > (get_source_lines): Likewise.
> > 
> > gcc/testsuite/ChangeLog:
> > 
> > * c-c++-common/diagnostic-format-sarif-file-5.c: New test.
> 
> I'm not sure if generated data is allowed as part of a SARIF artefact,
> or if there's a more standard-compliant way of representing this; SARIF
> says an artefact is a "sequence of bytes addressable via a URI".
> 
> Can you post a simple example of the generated .sarif JSON please? 
> e.g. from the new test, so that we can see it looks like.
> 
> You could run it through:
> 
>   python -m json.tool 
> 
> to format it for easier reading.

For a simple example like:

_Pragma("GCC diagnostic ignored \"-Wnot-an-option\"")

for which the normal output is:

=
In buffer generated from t.cpp:1:
:1:24: warning: unknown option after ‘#pragma GCC diagnostic’ kind 
[-Wpragmas]
1 | GCC diagnostic ignored "-Wnot-an-option"
  |^
t.cpp:1:1: note: in <_Pragma directive>
1 | _Pragma("GCC diagnostic ignored \"-Wnot-an-option\"")
  | ^~~
=

The SARIF output does not end up referencing any generated data locations,
because those are logically part of the "expansion" of the _Pragma
directive, and it doesn't output macro expansions.  In order for SARIF to
currently do something with generated data, it needs to see a generated data
location in a non-macro context. The only way to get GCC to do that, right
now, is with -fdump-internal-locations, which is what the new test case
does. That just unfortunately generates a larger amount of output. I attached
it, in case that's still helpful, for the following program:

=
_Pragma("GCC diagnostic push")
=

I guess there's potentially already a problem here because 'python -m
json.tool' is unhappy with this output and refuses to process it:

=
Invalid \escape: line 1 column 3436 (char 3435)
=

The related text is:
=
{"location": {"uri": "", "uriBaseId": "PWD"},
"contents":{"text": "GCC diagnostic push\n\0"}
=

And the \0 is not allowed it seems?

I also attached the output of 'python -m json.tool' anyway, after manually
removing the \0.

Is it better to just skip these locations for now?

-Lewis
{"$schema": 
"https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json";,
 "version": "2.1.0", "runs": [{"tool": {"driver": {"name": "GNU C++17", 
"fullName": "GNU C++17 (GCC) version 14.0.0 20230811 (experimental) 
(x86_64-pc-linux-gnu)", "version": "14.0.0 20230811 (experimental)", 
"informationUri": "https://gcc.gnu.org/gcc-14/";, "rules": []}}, "invocations": 
[{"executionSuccessful": true, "toolExecutionNotifications": []}], 
"originalUriBaseIds": {"PWD": {"uri": "file:///home/lewis/"}}, "artifacts": 
[{"location": {"uri": "t.cpp", "uriBaseId": "PWD"}, "contents": {"text": 
"_Pragma(\"GCC diagnostic push\")\n"}, "sourceLanguage": "cplusplus"}, 
{"location": {"uri": "/usr/include/stdc-predef.h"}, "contents": {"text&

Re: [PATCH v4 2/8] libcpp: diagnostics: Support generated data in expanded locations

[Lewis Hyatt via Gcc-patches]

On Fri, Aug 11, 2023 at 07:02:49PM -0400, David Malcolm wrote:
> On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> > The previous patch in this series introduced the concept of LC_GEN line
> > maps. This patch continues on the path to using them to improve _Pragma
> > diagnostics, by adding a new source_id SRC member to struct
> > expanded_location, which is populated by linemap_expand_location. This
> > member allows call sites to detect and handle when a location refers to
> > generated data rather than a plain file name.
> > 
> > The previous FILE member of expanded_location is preserved (although
> > redundant with SRC), so that call sites which do not and never will care
> > about generated data do not need to be concerned about it. Call sites that
> > will care are modified here, to use SRC rather than FILE for comparing
> > locations.
> 
> Thanks; this seems like a good approach.
> 
> 
> [...snip...]
> 
> > diff --git a/gcc/edit-context.cc b/gcc/edit-context.cc
> > index 6f5bc6b9d8f..15052aec417 100644
> > --- a/gcc/edit-context.cc
> > +++ b/gcc/edit-context.cc
> > @@ -295,7 +295,7 @@ edit_context::apply_fixit (const fixit_hint *hint)
> >  {
> >expanded_location start = expand_location (hint->get_start_loc ());
> >expanded_location next_loc = expand_location (hint->get_next_loc ());
> > -  if (start.file != next_loc.file)
> > +  if (start.src != next_loc.src || start.src.is_buffer ())
> >  return false;
> >if (start.line != next_loc.line)
> >  return false;
> 
> Thinking about fix-it hints, it makes sense to reject attempts to
> create fix-it hints within generated strings, as we can't apply them or
> visualize them.
> 
> Does anywhere in the patch kit do that?  Either of 
>   rich_location::maybe_add_fixit
> or
>   rich_location::reject_impossible_fixit
> would be good places to do that.
>

So rich_location::reject_impossible_fixit does reject them for _Pragmas now,
because what the frontend sees and passes to it is a virtual location, and it
always rejects virtual locations. But it doesn't reject arbitrary generated
data locations that may be created in an ordinary non-virtual location. I
think it's this one-line change to reject those:

-- >8 --

diff --git a/libcpp/line-map.cc b/libcpp/line-map.cc
index 835e8e1b8cd..382594637ad 100644
--- a/libcpp/line-map.cc
+++ b/libcpp/line-map.cc
@@ -2545,7 +2545,8 @@ rich_location::maybe_add_fixit (location_t start,
 = linemap_client_expand_location_to_spelling_point (next_loc,
LOCATION_ASPECT_START);
   /* They must be within the same file...  */
-  if (exploc_start.src != exploc_next_loc.src)
+  if (exploc_start.src != exploc_next_loc.src
+  || exploc_start.src.is_buffer ())
 {
   stop_supporting_fixits ();
   return;

-- >8 --

However, there are many selftests in diagnostic-show-locus.cc that actually
verify we generate the fixit hints for generated data, so I would need also to
change those to skip the test in this case as well. That looks like this:

-- >8 --

diff --git a/gcc/diagnostic-show-locus.cc b/gcc/diagnostic-show-locus.cc
index 62c60645e88..884c55e91e9 100644
--- a/gcc/diagnostic-show-locus.cc
+++ b/gcc/diagnostic-show-locus.cc
@@ -3824,6 +3824,8 @@ test_diagnostic_show_locus_one_liner (const 
line_table_case &case_)
   test_one_liner_simple_caret ();
   test_one_liner_caret_and_range ();
   test_one_liner_multiple_carets_and_ranges ();
+  if (!ltt.m_generated_data)
+{
   test_one_liner_fixit_insert_before ();
   test_one_liner_fixit_insert_after ();
   test_one_liner_fixit_remove ();
@@ -3835,6 +3837,7 @@ test_diagnostic_show_locus_one_liner (const 
line_table_case &case_)
   test_one_liner_many_fixits_2 ();
   test_one_liner_labels ();
 }
+}

 /* Version of all one-liner tests exercising multibyte awareness.  For
simplicity we stick to using two multibyte characters in the test, U+1F602
@@ -4419,6 +4422,8 @@ test_diagnostic_show_locus_one_liner_utf8 (const 
line_table_case &case_)
   test_one_liner_simple_caret_utf8 ();
   test_one_liner_caret_and_range_utf8 ();
   test_one_liner_multiple_carets_and_ranges_utf8 ();
+  if (!ltt.m_generated_data)
+{
   test_one_liner_fixit_insert_before_utf8 ();
   test_one_liner_fixit_insert_after_utf8 ();
   test_one_liner_fixit_remove_utf8 ();
@@ -4428,6 +4433,7 @@ test_diagnostic_show_locus_one_liner_utf8 (const 
line_table_case &case_)
   test_one_liner_fixit_validation_adhoc_locations_utf8 ();
   test_one_liner_many_fixits_1_utf8 ();
   test_one_liner_many_fixits_2_utf8 ();
+}
   test_one_liner_labels_utf8 ();
   test_one_liner_colorized_utf8 ();
 }
@@ -5726,15 +5732,15 @@

Re: [PATCH v4 1/8] libcpp: Add LC_GEN linemaps to support in-memory buffers

[Lewis Hyatt via Gcc-patches]

On Fri, Aug 11, 2023 at 06:45:31PM -0400, David Malcolm wrote:
> On Wed, 2023-08-09 at 18:14 -0400, Lewis Hyatt wrote:
> 
> Hi Lewis, thanks for the patch...
> 
> > Add a new linemap reason LC_GEN which enables encoding the location of data
> > that was generated during compilation and does not appear in any source 
> > file.
> > There could be many use cases, such as, for instance, referring to the 
> > content
> > of builtin macros (not yet implemented, but an easy lift after this one.) 
> > The
> > first intended application is to create a place to store the input to a
> > _Pragma directive, so that proper locations can be assigned to those
> > tokens. This will be done in a subsequent commit.
> > 
> > The TO_FILE member of struct line_map_ordinary has been changed to a union
> > named SRC which can be either a file name, or a pointer to a line_map_data
> > struct describing the data. There is no space overhead added to the line
> > maps data structures.
> > 
> > Outside libcpp, this patch includes only the minimal changes implied by the
> > adjustment from TO_FILE to SRC in struct line_map_ordinary. Subsequent
> > patches will implement the new functionality.
> > 
> > libcpp/ChangeLog:
> > 
> > * include/line-map.h (enum lc_reason): Add LC_GEN.
> > (struct line_map_data): New struct.
> > (struct line_map_ordinary): Change TO_FILE from a char* to a union,
> > and rename to SRC.
> > (class source_id): New class.
> > (ORDINARY_MAP_GENERATED_DATA_P): New function.
> > (ORDINARY_MAP_GENERATED_DATA): New function.
> > (ORDINARY_MAP_GENERATED_DATA_LEN): New function.
> > (ORDINARY_MAP_SOURCE_ID): New function.
> > (ORDINARY_MAPS_SAME_FILE_P): New function.
> > (ORDINARY_MAP_CONTAINING_FILE_NAME): Declare.
> > (LINEMAP_FILE): Adapt to struct line_map_ordinary change.
> > (linemap_get_file_highest_location): Likewise.
> > * line-map.cc (source_id::operator==): New function.
> > (ORDINARY_MAP_CONTAINING_FILE_NAME): New function.
> > (linemap_add): Support creating LC_GEN maps.
> > (linemap_line_start): Support LC_GEN maps.
> > (linemap_check_files_exited): Likewise.
> > (linemap_position_for_loc_and_offset): Likewise.
> > (linemap_get_expansion_filename): Likewise.
> > (linemap_dump): Likewise.
> > (linemap_dump_location): Likewise.
> > (linemap_get_file_highest_location): Likewise.
> > * directives.cc (_cpp_do_file_change): Likewise.
> > 
> > gcc/c-family/ChangeLog:
> > 
> > * c-common.cc (try_to_locate_new_include_insertion_point): Recognize
> > and ignore LC_GEN maps.
> > 
> > gcc/cp/ChangeLog:
> > 
> > * module.cc (module_state::write_ordinary_maps): Recognize and
> > ignore LC_GEN maps, and adapt to interface change in struct
> > line_map_ordinary.
> > (module_state::read_ordinary_maps): Likewise.
> > 
> > gcc/ChangeLog:
> > 
> > * diagnostic-show-locus.cc (compatible_locations_p): Adapt to
> > interface change in struct line_map_ordinary.
> > * input.cc (special_fname_generated): New function.
> > (dump_location_info): Support LC_GEN maps.
> > (get_substring_ranges_for_loc): Adapt to interface change in struct
> > line_map_ordinary.
> > * input.h (special_fname_generated): Declare.
> > 
> > gcc/go/ChangeLog:
> > 
> > * go-linemap.cc (Gcc_linemap::to_string): Recognize and ignore
> > LC_GEN maps.
> > ---
> >  gcc/c-family/c-common.cc |  11 ++-
> >  gcc/cp/module.cc |   8 +-
> >  gcc/diagnostic-show-locus.cc |   2 +-
> >  gcc/go/go-linemap.cc |   3 +-
> >  gcc/input.cc |  27 +-
> >  gcc/input.h  |   1 +
> >  libcpp/directives.cc |   4 +-
> >  libcpp/include/line-map.h    | 144 
> >  libcpp/line-map.cc   | 181 +--
> >  9 files changed, 299 insertions(+), 82 deletions(-)
> 
> [...snip...]
> 
> > 
> > diff --git a/gcc/diagnostic-show-locus.cc b/gcc/diagnostic-show-locus.cc
> > index 0514815b51f..a2aa6b4e0b5 100644
> > --- a/gcc/diagnostic-show-locus.cc
> > +++ b/gcc/diagnostic-show-locus.cc
> > @@ -998,7 +998,7 @@ compatible_locations_p (location_t loc_a, location_t 
> > loc_b)
> >  are in the same

[PATCH v4 7/8] diagnostics: libcpp: Assign real locations to the tokens inside _Pragma strings

[Lewis Hyatt via Gcc-patches]

Currently, the tokens obtained from a destringified _Pragma string do not get
assigned proper locations while they are being lexed.  After the tokens have
been obtained, they are reassigned the same location as the _Pragma token,
which is sufficient to make things like _Pragma("GCC diagnostic ignored...")
operate correctly, but this still results in inferior diagnostics, since the
diagnostics do not point to the problematic tokens.  Further, if a diagnostic
is issued by libcpp during the lexing of the tokens, as opposed to being
issued by the frontend during the processing of the pragma, then the
patched-up location is not yet in place, and the user rather sees an invalid
location that is near to the location of the _Pragma string in some cases, or
potentially very far away, depending on the macro expansion history.  For
example:

=
_Pragma("GCC diagnostic ignored \"oops")
=

produces the diagnostic:

file.cpp:1:24: warning: missing terminating " character
1 | _Pragma("GCC diagnostic ignored \"oops")
  |^

with the caret in a nonsensical location, while this one:

=
 #define S "GCC diagnostic ignored \"oops"
_Pragma(S)
=

produces:

file.cpp:2:24: warning: missing terminating " character
2 | _Pragma(S)
  |^

with both the caret in a nonsensical location, and the actual relevant context
completely absent.

Fix this by assigning proper locations using the new LC_GEN type of linemap.
Now the tokens are given locations inside a generated content buffer, and the
macro expansion stack is modified to be aware that these tokens logically
belong to the "expansion" of the _Pragma directive. For the above examples we
now output:

==
In buffer generated from file.cpp:1:
:1:24: warning: missing terminating " character
1 | GCC diagnostic ignored "oops
  |^
file.cpp:1:1: note: in <_Pragma directive>
1 | _Pragma("GCC diagnostic ignored \"oops")
  | ^~~
==

and

==
:1:24: warning: missing terminating " character
1 | GCC diagnostic ignored "oops
  |^
file.cpp:2:1: note: in <_Pragma directive>
2 | _Pragma(S)
  | ^~~
==

So that carets are pointing to something meaningful and all relevant context
appears in the diagnostic.  For the second example, it would be nice if the
macro expansion also output "in expansion of macro S", however doing that for
a general case of macro expansions makes the logic very complicated, since it
has to be done after the fact when the macro maps have already been
constructed.  It doesn't seem worth it for this case, given that the _Pragma
string has already been output once on the first line.

gcc/ChangeLog:

* tree-diagnostic.cc (maybe_unwind_expanded_macro_loc): Add awareness
of _Pragma directive to the macro expansion trace.

libcpp/ChangeLog:

* directives.cc (get_token_no_padding): Add argument to receive the
virtual location of the token.
(get__Pragma_string): Likewise.
(do_pragma): Set pfile->directive_result->src_loc properly, it should
not be a virtual location.
(destringize_and_run): Update to provide proper locations for the
_Pragma string tokens.  Support raw strings.
(_cpp_do__Pragma): Adapt to changes to the helper functions.
* errors.cc (cpp_diagnostic_at): Support
cpp_reader::diagnostic_rebase_loc.
(cpp_diagnostic_with_line): Likewise.
* include/line-map.h (class rich_location): Add new member
forget_cached_expanded_locations().
* internal.h (struct _cpp__Pragma_state): Define new struct.
(_cpp_rebase_diagnostic_location): Declare new function.
(struct cpp_reader): Add diagnostic_rebase_loc member.
(_cpp_push__Pragma_token_context): Declare new function.
(_cpp_do__Pragma): Adjust prototype.
* macro.cc (pragma_str): New static var.
(builtin_macro): Adapt to new implementation of _Pragma processing.
(_cpp_pop_context): Fix the logic for resetting
pfile->top_most_macro_node, which previously was never triggered,
although the error seems to have been harmless.
(_cpp_push__Pragma_token_context): New function.
(_cpp_rebase_diagnostic_location): New function.

gcc/c-family/ChangeLog:

* c-ppoutput.cc (token_streamer::stream): Pass the virtual location of
the _Pragma token to maybe_print_line(), not the spelling location.

libgomp/ChangeLog:

* testsuite/libgomp.oacc-c-c++-common/reduction-5.c: Adjust for new
macro tracking output for _Pragma directives.
* testsuite/libgomp.oacc-c-c++-common/vred2d-128.c: Likewise.

gcc/testsuite/ChangeLog:

* c-c++-common/cpp/diagnostic-pragma-1.c: Adjust for new macro
tracking output for _Pragma directives.
* c-c++-common/cpp/pr57580.c: Likewise.
* c-c++-common/gomp/pragma-3.c: Likewise.

[PATCH v4 6/8] diagnostics: Full support for generated data locations

[Lewis Hyatt via Gcc-patches]

Previous patches in this series have laid the groundwork for supporting
source code locations in memory ("generated data") rather than ordinary
files. This patch completes the support by adding awareness of such
locations to all places that need to support them. The main changes are to
diagnostic-show-locus.cc; the others are primarily small tweaks such as
changing from the FILE to the SRC member when inspecting an
expanded_location.

gcc/c-family/ChangeLog:

* c-format.cc (get_corrected_substring): Use the new overload of
location_get_source_line() to support generated data.
* c-indentation.cc (get_visual_column): Likewise.
(get_first_nws_vis_column): Change argument from a plain file name
to a source_id.
(detect_intervening_unindent): Likewise.
(should_warn_for_misleading_indentation): Pass
detect_intervening_unindent() the SRC field rather than the FILE
field from the expanded_location.

gcc/ChangeLog:

* gcc-rich-location.cc (blank_line_before_p): Use the new overload
of location_get_source_line() to support generated data.
* input.cc (get_source_text_between): Likewise.
(get_substring_ranges_for_loc): Likewise.
(get_source_file_content): Change the argument from a plain filename
to a source_id.
(location_missing_trailing_newline): Likewise.
* input.h (get_source_file_content): Adjust prototype.
(location_missing_trailing_newline): Likewise.
* diagnostic-show-locus.cc (layout::calculate_x_offset_display): Use
the new overload of location_get_source_line() to support generated
data.
(layout::print_line): Likewise.
(class line_corrections): Change m_filename from a plain filename to
a source_id.
(source_line::source_line): Change argument from a plain filename to
a source_id.
(line_corrections::add_hint): Adapt to source_line change.
(layout::print_trailing_fixits): Adapt to line_corrections change.
(test_layout_x_offset_display_utf8): Test generated data too.
(test_layout_x_offset_display_tab): Likewise.
(test_diagnostic_show_locus_one_liner): Likewise.
(test_diagnostic_show_locus_one_liner_utf8): Likewise.
(test_add_location_if_nearby): Likewise.
(test_diagnostic_show_locus_fixit_lines): Likewise.
(test_fixit_consolidation): Likewise.
(test_overlapped_fixit_printing): Likewise.
(test_overlapped_fixit_printing_utf8): Likewise.
(test_overlapped_fixit_printing_2): Likewise.
(test_fixit_insert_containing_newline): Likewise.
(test_fixit_insert_containing_newline_2): Likewise.
(test_fixit_replace_containing_newline): Likewise.
(test_fixit_deletion_affecting_newline): Likewise.
(test_tab_expansion): Likewise.
(test_escaping_bytes_1): Likewise.
(test_escaping_bytes_2): Likewise.
(test_line_numbers_multiline_range): Likewise.
(diagnostic_show_locus_cc_tests): Likewise.
---
 gcc/c-family/c-format.cc  |   2 +-
 gcc/c-family/c-indentation.cc |   8 +-
 gcc/diagnostic-show-locus.cc  | 227 ++
 gcc/gcc-rich-location.cc  |   2 +-
 gcc/input.cc  |  21 ++--
 gcc/input.h   |   6 +-
 6 files changed, 136 insertions(+), 130 deletions(-)

diff --git a/gcc/c-family/c-format.cc b/gcc/c-family/c-format.cc
index 529b1408179..929ec24622c 100644
--- a/gcc/c-family/c-format.cc
+++ b/gcc/c-family/c-format.cc
@@ -4537,7 +4537,7 @@ get_corrected_substring (const substring_loc &fmt_loc,
   if (caret.column > finish.column)
 return NULL;
 
-  char_span line = location_get_source_line (start.file, start.line);
+  char_span line = location_get_source_line (start);
   if (!line)
 return NULL;
 
diff --git a/gcc/c-family/c-indentation.cc b/gcc/c-family/c-indentation.cc
index fce74991aae..27a90d9cc15 100644
--- a/gcc/c-family/c-indentation.cc
+++ b/gcc/c-family/c-indentation.cc
@@ -50,7 +50,7 @@ get_visual_column (expanded_location exploc,
   unsigned int *first_nws,
   unsigned int tab_width)
 {
-  char_span line = location_get_source_line (exploc.file, exploc.line);
+  char_span line = location_get_source_line (exploc);
   if (!line)
 return false;
   if ((size_t)exploc.column > line.length ())
@@ -87,7 +87,7 @@ get_visual_column (expanded_location exploc,
Otherwise, return false, leaving *FIRST_NWS untouched.  */
 
 static bool
-get_first_nws_vis_column (const char *file, int line_num,
+get_first_nws_vis_column (source_id file, int line_num,
  unsigned int *first_nws,
  unsigned int tab_width)
 {
@@ -158,7 +158,7 @@ get_first_nws_vis_column (const char *file, int line_num,
Return true if such an unindent/outdent is detected.  */
 
 static bool
-detect_intervening_unindent (const char *file,
+detect_in

[PATCH v4 5/8] diagnostics: Support testing generated data in input.cc selftests

[Lewis Hyatt via Gcc-patches]

Add selftests for the new capabilities in input.cc related to source code
locations that are stored in memory rather than ordinary files.

gcc/ChangeLog:

* input.cc (temp_source_file::do_linemap_add): New function.
(line_table_case::line_table_case): Add GENERATED_DATA argument.
(line_table_test::line_table_test): Implement new M_GENERATED_DATA
argument.
(for_each_line_table_case): Optionally include generated data
locations in the set of cases.
(test_accessing_ordinary_linemaps): Test generated data locations.
(test_make_location_nonpure_range_endpoints): Likewise.
(test_line_offset_overflow): Likewise.
(input_cc_tests): Likewise.
* selftest.cc (named_temp_file::named_temp_file): Interpret a null
SUFFIX argument as a request to use in-memory data.
(named_temp_file::~named_temp_file): Support in-memory data.
(temp_source_file::temp_source_file): Likewise.
(temp_source_file::~temp_source_file): Likewise.
* selftest.h (struct line_map_ordinary): Foward declare.
(class named_temp_file): Add missing explicit to the constructor.
(class temp_source_file): Add new members to support in-memory data.
(class line_table_test): Likewise.
(for_each_line_table_case): Adjust prototype.
---
 gcc/input.cc| 81 +
 gcc/selftest.cc | 53 +---
 gcc/selftest.h  | 19 ++--
 3 files changed, 113 insertions(+), 40 deletions(-)

diff --git a/gcc/input.cc b/gcc/input.cc
index 790279d4273..8c4e40aaf23 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -2066,6 +2066,20 @@ get_num_source_ranges_for_substring (cpp_reader *pfile,
 
 /* Selftests of location handling.  */
 
+/* Wrapper around linemap_add to handle transparently adding either a tmp file,
+   or in-memory generated content.  */
+const line_map_ordinary *
+temp_source_file::do_linemap_add (int line)
+{
+  const line_map *map;
+  if (content_buf)
+map = linemap_add (line_table, LC_GEN, false, content_buf,
+  line, content_len);
+  else
+map = linemap_add (line_table, LC_ENTER, false, get_filename (), line);
+  return linemap_check_ordinary (map);
+}
+
 /* Verify that compare() on linenum_type handles comparisons over the full
range of the type.  */
 
@@ -2144,13 +2158,16 @@ assert_loceq (const char *exp_filename, int 
exp_linenum, int exp_colnum,
 class line_table_case
 {
 public:
-  line_table_case (int default_range_bits, int base_location)
+  line_table_case (int default_range_bits, int base_location,
+  bool generated_data)
   : m_default_range_bits (default_range_bits),
-m_base_location (base_location)
+m_base_location (base_location),
+m_generated_data (generated_data)
   {}
 
   int m_default_range_bits;
   int m_base_location;
+  bool m_generated_data;
 };
 
 /* Constructor.  Store the old value of line_table, and create a new
@@ -2167,6 +2184,7 @@ line_table_test::line_table_test ()
   gcc_assert (saved_line_table->round_alloc_size);
   line_table->round_alloc_size = saved_line_table->round_alloc_size;
   line_table->default_range_bits = 0;
+  m_generated_data = false;
 }
 
 /* Constructor.  Store the old value of line_table, and create a new
@@ -2188,6 +2206,7 @@ line_table_test::line_table_test (const line_table_case 
&case_)
   line_table->highest_location = case_.m_base_location;
   line_table->highest_line = case_.m_base_location;
 }
+  m_generated_data = case_.m_generated_data;
 }
 
 /* Destructor.  Restore the old value of line_table.  */
@@ -2207,7 +2226,10 @@ test_accessing_ordinary_linemaps (const line_table_case 
&case_)
   line_table_test ltt (case_);
 
   /* Build a simple linemap describing some locations. */
-  linemap_add (line_table, LC_ENTER, false, "foo.c", 0);
+  if (ltt.m_generated_data)
+linemap_add (line_table, LC_GEN, false, "some data", 0, 10);
+  else
+linemap_add (line_table, LC_ENTER, false, "foo.c", 0);
 
   linemap_line_start (line_table, 1, 100);
   location_t loc_a = linemap_position_for_column (line_table, 1);
@@ -2257,21 +2279,23 @@ test_accessing_ordinary_linemaps (const line_table_case 
&case_)
   linemap_add (line_table, LC_LEAVE, false, NULL, 0);
 
   /* Verify that we can recover the location info.  */
-  assert_loceq ("foo.c", 1, 1, loc_a);
-  assert_loceq ("foo.c", 1, 23, loc_b);
-  assert_loceq ("foo.c", 2, 1, loc_c);
-  assert_loceq ("foo.c", 2, 17, loc_d);
-  assert_loceq ("foo.c", 3, 700, loc_e);
-  assert_loceq ("foo.c", 4, 100, loc_back_to_short);
+  const auto fname
+= (ltt.m_generated_data ? special_fname_generated () : "foo.c");
+  assert_loceq (fname, 1, 1, loc_a);
+  assert_loceq (fname, 1, 23, loc_b);
+  assert_loceq (fname, 2, 1, loc_c);
+  assert_loceq (fname, 2, 17, loc_d);
+  assert_loceq (fname, 3, 700, loc_e);
+  assert_loceq (fname, 4, 100, loc_back_to_short);
 
   /* In the very wide li

[PATCH v4 3/8] diagnostics: Refactor class file_cache_slot

[Lewis Hyatt via Gcc-patches]

Class file_cache_slot in input.cc is used to query specific lines of source
code from a file when needed by diagnostics infrastructure. This will be
extended in a subsequent patch to support obtaining the source code from
in-memory generated buffers rather than from a file. The present patch
refactors class file_cache_slot, putting most of the logic into a new base
class cache_data_source, in preparation for reusing that code in the next
patch. There is no change in functionality yet.

gcc/ChangeLog:

* input.cc (class file_cache_slot): Refactor functionality into a
new base class...
(class cache_data_source): ...here.
(file_cache::forcibly_evict_file): Adapt for refactoring.
(file_cache_slot::evict): Renamed to...
(file_cache_slot::reset): ...this, and partially refactored into
base class...
(cache_data_source::reset): ...here.
(file_cache_slot::get_full_file_content): Moved into base class...
(cache_data_source::get_full_file_content): ...here.
(file_cache_slot::create): Adapt for refactoring.
(file_cache_slot::file_cache_slot): Refactor partially into...
(cache_data_source::cache_data_source): ...here.
(file_cache_slot::~file_cache_slot): Refactor partially into...
(cache_data_source::~cache_data_source): ...here.
(file_cache_slot::needs_read_p): Remove.
(file_cache_slot::needs_grow_p): Remove.
(file_cache_slot::maybe_grow): Adapt for refactoring.
(file_cache_slot::read_data): Refactored, along with...
(file_cache_slot::maybe_read_data): this, into...
(file_cache_slot::get_more_data): ...here.
(find_end_of_line): Change interface to take a pair of pointers,
rather than a pointer + length.
(file_cache_slot::get_next_line): Refactored into...
(cache_data_source::get_next_line): ...here.
(file_cache_slot::goto_next_line): Refactored into...
(cache_data_source::goto_next_line): ...here.
(file_cache_slot::read_line_num): Refactored into...
(cache_data_source::read_line_num): ...here.
(location_get_source_line): Fix const-correctness as necessitated by
new interface.
---
 gcc/input.cc | 513 +++
 1 file changed, 235 insertions(+), 278 deletions(-)

diff --git a/gcc/input.cc b/gcc/input.cc
index c2559614a99..9377020b460 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -55,34 +55,88 @@ file_cache::initialize_input_context 
(diagnostic_input_charset_callback ccb,
   in_context.should_skip_bom = should_skip_bom;
 }
 
-/* This is a cache used by get_next_line to store the content of a
-   file to be searched for file lines.  */
-class file_cache_slot
+/* This is an abstract interface for a class that provides data which we want 
to
+   look up by line number.  Concrete implementations will follow, which handle
+   the cases of reading the data from the input source files, or of reading it
+   from in-memory generated data buffers.  The design is driven with reading
+   from files in mind, in particular it is desirable to read only as much of a
+   file from disk as necessary.  It works like a simplified std::istream, i.e.
+   virtual function calls are only needed when we need to retrieve more data
+   from the underlying source.  */
+
+class cache_data_source
 {
-public:
-  file_cache_slot ();
-  ~file_cache_slot ();
 
-  bool read_line_num (size_t line_num,
- char ** line, ssize_t *line_len);
-
-  /* Accessors.  */
-  const char *get_file_path () const { return m_file_path; }
+public:
+  bool read_line_num (size_t line_num, const char **line, ssize_t *line_len);
   unsigned get_use_count () const { return m_use_count; }
+  void inc_use_count () { m_use_count++; }
+  bool get_next_line (const char **line, ssize_t *line_len);
+  bool goto_next_line ();
   bool missing_trailing_newline_p () const
   {
 return m_missing_trailing_newline;
   }
   char_span get_full_file_content ();
+  bool unused () const { return !m_data_begin; }
+  virtual void reset ();
+
+protected:
+  cache_data_source ();
+  virtual ~cache_data_source ();
+
+  /* These pointers delimit the data that we are processing.  They are
+ maintained by the derived classes, we only ask for more by calling
+ get_more_data().  That function should return TRUE if more data was
+ obtained.  Calling get_more_data () may invalidate these pointers
+ (i.e. reallocating them to a larger buffer).  */
+  const char *m_data_begin;
+  const char *m_data_end;
+  virtual bool get_more_data () = 0;
+
+  /* This is to be called by the derived classes when this object is
+ being activated.  */
+  void on_create (unsigned int use_count, size_t total_lines)
+  {
+m_use_count = use_count;
+m_total_lines = total_lines;
+  }
 
-  void inc_use_count () { m_use_count++; }
+private:
+  /* Non-copyable.  */
+  cache_data_source (const cache_

[PATCH v4 4/8] diagnostics: Support obtaining source code lines from generated data buffers

[Lewis Hyatt via Gcc-patches]

This patch enhances location_get_source_line(), which is the primary
interface provided by the diagnostics infrastructure to obtain the line of
source code corresponding to a given location, so that it understands
generated data locations in addition to normal file-based locations. This
involves changing the argument to location_get_source_line() from a plain
file name, to a source_id object that can represent either type of location.

gcc/ChangeLog:

* input.cc (class data_cache_slot): New class.
(file_cache::lookup_data): New function.
(diagnostics_file_cache_forcibly_evict_data): New function.
(file_cache::forcibly_evict_data): New function.
(file_cache::evicted_cache_tab_entry): Generalize (via a template)
to work for both file_cache_slot and data_cache_slot.
(file_cache::add_file): Adapt for new interface to
evicted_cache_tab_entry.
(file_cache::add_data): New function.
(data_cache_slot::create): New function.
(file_cache::file_cache): Support the new m_data_slots member.
(file_cache::~file_cache): Likewise.
(file_cache::lookup_or_add_data): New function.
(file_cache::lookup_or_add): New function that calls either
lookup_or_add_data or lookup_or_add_file as appropriate.
(location_get_source_line): Change the FILE_PATH argument to a
source_id SRC, and use it to support obtaining source lines from
generated data as well as from files.
(location_compute_display_column): Support generated data using the
new features of location_get_source_line.
(dump_location_info): Likewise.
* input.h (location_get_source_line): Adjust prototype. Add a new
convenience overload taking an expanded_location.
(class cache_data_source): Declare.
(class data_cache_slot): Declare.
(class file_cache): Declare new members.
(diagnostics_file_cache_forcibly_evict_data): Declare.
---
 gcc/input.cc | 171 ---
 gcc/input.h  |  23 +--
 2 files changed, 153 insertions(+), 41 deletions(-)

diff --git a/gcc/input.cc b/gcc/input.cc
index 9377020b460..790279d4273 100644
--- a/gcc/input.cc
+++ b/gcc/input.cc
@@ -207,6 +207,28 @@ private:
   void maybe_grow ();
 };
 
+/* This is the implementation of cache_data_source for generated
+   data that is already in memory.  */
+class data_cache_slot final : public cache_data_source
+{
+public:
+  void create (const char *data, unsigned int data_len,
+  unsigned int highest_use_count);
+  bool represents_data (const char *data, unsigned int) const
+  {
+/* We can just use pointer equality here since the generated data lives in
+   memory in one persistent place.  It isn't anticipated there would be
+   several generated data buffers with the same content, so we don't mind
+   that in such a case we will store it twice.  */
+return m_data_begin == data;
+  }
+
+protected:
+  /* In contrast to file_cache_slot, we do not own a buffer.  The buffer
+ passed to create() needs to outlive this object.  */
+  bool get_more_data () override { return false; }
+};
+
 /* Current position in real source file.  */
 
 location_t input_location = UNKNOWN_LOCATION;
@@ -382,6 +404,21 @@ file_cache::lookup_file (const char *file_path)
   return r;
 }
 
+data_cache_slot *
+file_cache::lookup_data (const char *data, unsigned int data_len)
+{
+  for (unsigned int i = 0; i != num_file_slots; ++i)
+{
+  const auto slot = m_data_slots + i;
+  if (slot->represents_data (data, data_len))
+   {
+ slot->inc_use_count ();
+ return slot;
+   }
+}
+  return nullptr;
+}
+
 /* Purge any mention of FILENAME from the cache of files used for
printing source code.  For use in selftests when working
with tempfiles.  */
@@ -397,6 +434,15 @@ diagnostics_file_cache_forcibly_evict_file (const char 
*file_path)
   global_dc->m_file_cache->forcibly_evict_file (file_path);
 }
 
+void
+diagnostics_file_cache_forcibly_evict_data (const char *data,
+   unsigned int data_len)
+{
+  if (!global_dc->m_file_cache)
+return;
+  global_dc->m_file_cache->forcibly_evict_data (data, data_len);
+}
+
 void
 file_cache::forcibly_evict_file (const char *file_path)
 {
@@ -410,36 +456,36 @@ file_cache::forcibly_evict_file (const char *file_path)
   r->reset ();
 }
 
+void
+file_cache::forcibly_evict_data (const char *data, unsigned int data_len)
+{
+  if (auto r = lookup_data (data, data_len))
+r->reset ();
+}
+
 /* Return the cache that has been less used, recently, or the
first empty one.  If HIGHEST_USE_COUNT is non-null,
*HIGHEST_USE_COUNT is set to the highest use count of the entries
in the cache table.  */
 
-file_cache_slot*
-file_cache::evicted_cache_tab_entry (unsigned *highest_use_count)
+template 
+Slot *
+file_cache::evicted_cache_tab_entry (Slot *

[PATCH v4 8/8] diagnostics: Support generated data locations in SARIF output

[Lewis Hyatt via Gcc-patches]

The diagnostics routines for SARIF output need to read the source code back
in, so that they can generate "snippet" and "content" records, so they need to
be able to cope with generated data locations.  Add support for that in
diagnostic-format-sarif.cc.

gcc/ChangeLog:

* diagnostic-format-sarif.cc (class sarif_builder): Adapt interface
to support generated data locations.
(sarif_builder::maybe_make_physical_location_object): Change the
m_filenames hash_set to support generated data.
(sarif_builder::make_artifact_location_object): Use a source_id rather
than a plain file name.
(sarif_builder::maybe_make_region_object): Adapt to
expanded_location interface changes.
(sarif_builder::maybe_make_region_object_for_context): Likewise.
(sarif_builder::make_artifact_object): Likewise.
(sarif_builder::make_run_object): Handle generated data.
(sarif_builder::maybe_make_artifact_content_object): Likewise.
(get_source_lines): Likewise.

gcc/testsuite/ChangeLog:

* c-c++-common/diagnostic-format-sarif-file-5.c: New test.
---
 gcc/diagnostic-format-sarif.cc| 88 +++
 .../diagnostic-format-sarif-file-5.c  | 31 +++
 2 files changed, 82 insertions(+), 37 deletions(-)
 create mode 100644 gcc/testsuite/c-c++-common/diagnostic-format-sarif-file-5.c

diff --git a/gcc/diagnostic-format-sarif.cc b/gcc/diagnostic-format-sarif.cc
index 1eff71962d7..c7c0e5d4b0a 100644
--- a/gcc/diagnostic-format-sarif.cc
+++ b/gcc/diagnostic-format-sarif.cc
@@ -174,7 +174,7 @@ private:
   json::array *maybe_make_kinds_array (diagnostic_event::meaning m) const;
   json::object *maybe_make_physical_location_object (location_t loc);
   json::object *make_artifact_location_object (location_t loc);
-  json::object *make_artifact_location_object (const char *filename);
+  json::object *make_artifact_location_object (source_id src);
   json::object *make_artifact_location_object_for_pwd () const;
   json::object *maybe_make_region_object (location_t loc) const;
   json::object *maybe_make_region_object_for_context (location_t loc) const;
@@ -197,9 +197,9 @@ private:
   json::object *make_reporting_descriptor_object_for_cwe_id (int cwe_id) const;
   json::object *
   make_reporting_descriptor_reference_object_for_cwe_id (int cwe_id);
-  json::object *make_artifact_object (const char *filename);
-  json::object *maybe_make_artifact_content_object (const char *filename) 
const;
-  json::object *maybe_make_artifact_content_object (const char *filename,
+  json::object *make_artifact_object (source_id src);
+  json::object *maybe_make_artifact_content_object (source_id src) const;
+  json::object *maybe_make_artifact_content_object (source_id src,
int start_line,
int end_line) const;
   json::object *make_fix_object (const rich_location &rich_loc);
@@ -220,7 +220,11 @@ private:
  diagnostic group.  */
   sarif_result *m_cur_group_result;
 
-  hash_set  m_filenames;
+  /* If the second member is >0, then this is a buffer of generated content,
+ with that length, not a filename.  */
+  hash_set ,
+  int_hash  >
+   > m_filenames;
   bool m_seen_any_relative_paths;
   hash_set  m_rule_id_set;
   json::array *m_rules_arr;
@@ -787,7 +791,8 @@ sarif_builder::maybe_make_physical_location_object 
(location_t loc)
   /* "artifactLocation" property (SARIF v2.1.0 section 3.29.3).  */
   json::object *artifact_loc_obj = make_artifact_location_object (loc);
   phys_loc_obj->set ("artifactLocation", artifact_loc_obj);
-  m_filenames.add (LOCATION_FILE (loc));
+  const auto src = LOCATION_SRC (loc);
+  m_filenames.add ({src.get_filename_or_buffer (), src.get_buffer_len ()});
 
   /* "region" property (SARIF v2.1.0 section 3.29.4).  */
   if (json::object *region_obj = maybe_make_region_object (loc))
@@ -811,7 +816,7 @@ sarif_builder::maybe_make_physical_location_object 
(location_t loc)
 json::object *
 sarif_builder::make_artifact_location_object (location_t loc)
 {
-  return make_artifact_location_object (LOCATION_FILE (loc));
+  return make_artifact_location_object (LOCATION_SRC (loc));
 }
 
 /* The ID value for use in "uriBaseId" properties (SARIF v2.1.0 section 3.4.4)
@@ -823,10 +828,13 @@ sarif_builder::make_artifact_location_object (location_t 
loc)
or return NULL.  */
 
 json::object *
-sarif_builder::make_artifact_location_object (const char *filename)
+sarif_builder::make_artifact_location_object (source_id src)
 {
   json::object *artifact_loc_obj = new json::object ();
 
+  const auto filename = src.is_buffer ()
+? special_fname_generated () : src.get_filename_or_buffer ();
+
   /* "uri" property (SARIF v2.1.0 section 3.4.3).  */
   artifact_loc_obj->set ("uri", new json::string (filename));
 
@@ -912,9 +920,9 @@ sarif_builder::maybe_make_region_object (location_t l

[PATCH v4 1/8] libcpp: Add LC_GEN linemaps to support in-memory buffers

[Lewis Hyatt via Gcc-patches]

Add a new linemap reason LC_GEN which enables encoding the location of data
that was generated during compilation and does not appear in any source file.
There could be many use cases, such as, for instance, referring to the content
of builtin macros (not yet implemented, but an easy lift after this one.) The
first intended application is to create a place to store the input to a
_Pragma directive, so that proper locations can be assigned to those
tokens. This will be done in a subsequent commit.

The TO_FILE member of struct line_map_ordinary has been changed to a union
named SRC which can be either a file name, or a pointer to a line_map_data
struct describing the data. There is no space overhead added to the line
maps data structures.

Outside libcpp, this patch includes only the minimal changes implied by the
adjustment from TO_FILE to SRC in struct line_map_ordinary. Subsequent
patches will implement the new functionality.

libcpp/ChangeLog:

* include/line-map.h (enum lc_reason): Add LC_GEN.
(struct line_map_data): New struct.
(struct line_map_ordinary): Change TO_FILE from a char* to a union,
and rename to SRC.
(class source_id): New class.
(ORDINARY_MAP_GENERATED_DATA_P): New function.
(ORDINARY_MAP_GENERATED_DATA): New function.
(ORDINARY_MAP_GENERATED_DATA_LEN): New function.
(ORDINARY_MAP_SOURCE_ID): New function.
(ORDINARY_MAPS_SAME_FILE_P): New function.
(ORDINARY_MAP_CONTAINING_FILE_NAME): Declare.
(LINEMAP_FILE): Adapt to struct line_map_ordinary change.
(linemap_get_file_highest_location): Likewise.
* line-map.cc (source_id::operator==): New function.
(ORDINARY_MAP_CONTAINING_FILE_NAME): New function.
(linemap_add): Support creating LC_GEN maps.
(linemap_line_start): Support LC_GEN maps.
(linemap_check_files_exited): Likewise.
(linemap_position_for_loc_and_offset): Likewise.
(linemap_get_expansion_filename): Likewise.
(linemap_dump): Likewise.
(linemap_dump_location): Likewise.
(linemap_get_file_highest_location): Likewise.
* directives.cc (_cpp_do_file_change): Likewise.

gcc/c-family/ChangeLog:

* c-common.cc (try_to_locate_new_include_insertion_point): Recognize
and ignore LC_GEN maps.

gcc/cp/ChangeLog:

* module.cc (module_state::write_ordinary_maps): Recognize and
ignore LC_GEN maps, and adapt to interface change in struct
line_map_ordinary.
(module_state::read_ordinary_maps): Likewise.

gcc/ChangeLog:

* diagnostic-show-locus.cc (compatible_locations_p): Adapt to
interface change in struct line_map_ordinary.
* input.cc (special_fname_generated): New function.
(dump_location_info): Support LC_GEN maps.
(get_substring_ranges_for_loc): Adapt to interface change in struct
line_map_ordinary.
* input.h (special_fname_generated): Declare.

gcc/go/ChangeLog:

* go-linemap.cc (Gcc_linemap::to_string): Recognize and ignore
LC_GEN maps.
---
 gcc/c-family/c-common.cc |  11 ++-
 gcc/cp/module.cc |   8 +-
 gcc/diagnostic-show-locus.cc |   2 +-
 gcc/go/go-linemap.cc |   3 +-
 gcc/input.cc |  27 +-
 gcc/input.h  |   1 +
 libcpp/directives.cc |   4 +-
 libcpp/include/line-map.h| 144 
 libcpp/line-map.cc   | 181 +--
 9 files changed, 299 insertions(+), 82 deletions(-)

diff --git a/gcc/c-family/c-common.cc b/gcc/c-family/c-common.cc
index 9fbaeb437a1..ecfc2efc29f 100644
--- a/gcc/c-family/c-common.cc
+++ b/gcc/c-family/c-common.cc
@@ -9206,19 +9206,22 @@ try_to_locate_new_include_insertion_point (const char 
*file, location_t loc)
   const line_map_ordinary *ord_map
= LINEMAPS_ORDINARY_MAP_AT (line_table, i);
 
+  if (ORDINARY_MAP_GENERATED_DATA_P (ord_map))
+   continue;
+
   if (const line_map_ordinary *from
  = linemap_included_from_linemap (line_table, ord_map))
/* We cannot use pointer equality, because with preprocessed
   input all filename strings are unique.  */
-   if (0 == strcmp (from->to_file, file))
+   if (ORDINARY_MAP_SOURCE_ID (from) == file)
  {
last_include_ord_map = from;
last_ord_map_after_include = NULL;
  }
 
-  /* Likewise, use strcmp, and reject any line-zero introductory
-map.  */
-  if (ord_map->to_line && 0 == strcmp (ord_map->to_file, file))
+  /* Likewise, use strcmp (via the source_id comparison), and reject any
+line-zero introductory map.  */
+  if (ord_map->to_line && ORDINARY_MAP_SOURCE_ID (ord_map) == file)
{
  if (!first_ord_map_in_file)
first_ord_map_in_file = ord_map;
diff --git a/gcc/cp/module.cc b/gcc/cp/module.cc
index ea362bdffa4..ff17cd57016 100644
--- a/gcc/cp/

[PATCH v4 2/8] libcpp: diagnostics: Support generated data in expanded locations

[Lewis Hyatt via Gcc-patches]

The previous patch in this series introduced the concept of LC_GEN line
maps. This patch continues on the path to using them to improve _Pragma
diagnostics, by adding a new source_id SRC member to struct
expanded_location, which is populated by linemap_expand_location. This
member allows call sites to detect and handle when a location refers to
generated data rather than a plain file name.

The previous FILE member of expanded_location is preserved (although
redundant with SRC), so that call sites which do not and never will care
about generated data do not need to be concerned about it. Call sites that
will care are modified here, to use SRC rather than FILE for comparing
locations.

libcpp/ChangeLog:

* include/line-map.h (struct expanded_location): Add SRC member. Add
zero-initializers for all members, since source_id is not a POD
type.
(class fixit_hint): Adjust prototype.
* line-map.cc (linemap_expand_location): Populate the new SRC member
in the expanded_location.
(rich_location::maybe_add_fixit): Compare explocs with the new SRC
field instead of the FILE field.
(fixit_hint::affects_line_p): Accept a source_id instead of a file
name, and use it for the comparisons.

gcc/c-family/ChangeLog:

* c-format.cc (get_corrected_substring): Compare explocs with the
new SRC field instead of the FILE field.
* c-indentation.cc (should_warn_for_misleading_indentation): Likewise.
(assert_get_visual_column_succeeds): Initialize the SRC field in the
test expanded_location.
(assert_get_visual_column_fails): Likewise.

gcc/ChangeLog:

* diagnostic-show-locus.cc (make_range): Adapt to the new
constructor semantics for struct expanded_location.
(layout::maybe_add_location_range): Compare explocs with the new SRC
field instead of the FILE field.
(layout::validate_fixit_hint_p): Likewise.
(layout::print_leading_fixits): Use the SRC field in struct
expanded_location to query fixit_hint::affects_line_p.
(layout::print_trailing_fixits): Likewise.
* diagnostic.cc (diagnostic_report_current_module): Use the new SRC
field in expanded_location to detect LC_GEN locations and identify
them as such.
(assert_location_text): Adapt to the new constructor semantics for
struct expanded_location.
* input.cc (expand_location_1): Likewise. And when libcpp's
linemap_expand_location returns a null FILE for generated data,
replace it with special_fname_generated ().
(total_lines_num): Handle a generic source_id argument rather than a
file name only.
(get_source_text_between): Compare explocs with the new SRC field
instead of the FILE field.
(get_substring_ranges_for_loc): Likewise.
* edit-context.cc (edit_context::apply_fixit): Ignore locations in
generated data.
* input.h (LOCATION_SRC): New accessor macro.
---
 gcc/c-family/c-format.cc  |  4 ++--
 gcc/c-family/c-indentation.cc | 10 +-
 gcc/diagnostic-show-locus.cc  | 30 +-
 gcc/diagnostic.cc | 19 ---
 gcc/edit-context.cc   |  2 +-
 gcc/input.cc  | 21 +++--
 gcc/input.h   |  1 +
 libcpp/include/line-map.h | 24 ++--
 libcpp/line-map.cc| 15 +++
 9 files changed, 70 insertions(+), 56 deletions(-)

diff --git a/gcc/c-family/c-format.cc b/gcc/c-family/c-format.cc
index b4eeebcb30e..529b1408179 100644
--- a/gcc/c-family/c-format.cc
+++ b/gcc/c-family/c-format.cc
@@ -4522,9 +4522,9 @@ get_corrected_substring (const substring_loc &fmt_loc,
 = expand_location_to_spelling_point (fmt_substring_range.m_start);
   expanded_location finish
 = expand_location_to_spelling_point (fmt_substring_range.m_finish);
-  if (caret.file != start.file)
+  if (caret.src != start.src)
 return NULL;
-  if (start.file != finish.file)
+  if (start.src != finish.src)
 return NULL;
   if (caret.line != start.line)
 return NULL;
diff --git a/gcc/c-family/c-indentation.cc b/gcc/c-family/c-indentation.cc
index e8d3dece770..fce74991aae 100644
--- a/gcc/c-family/c-indentation.cc
+++ b/gcc/c-family/c-indentation.cc
@@ -334,7 +334,7 @@ should_warn_for_misleading_indentation (const 
token_indent_info &guard_tinfo,
   const unsigned int tab_width = global_dc->tabstop;
 
   /* They must be in the same file.  */
-  if (next_stmt_exploc.file != body_exploc.file)
+  if (next_stmt_exploc.src != body_exploc.src)
 return false;
 
   /* If NEXT_STMT_LOC and BODY_LOC are on the same line, consider
@@ -363,7 +363,7 @@ should_warn_for_misleading_indentation (const 
token_indent_info &guard_tinfo,
   ^ DON'T WARN HERE.  */
   if (next_stmt_exploc.line == body_exploc.line)
 {
-  if (guard_explo

[PATCH v4 0/8] diagnostics: libcpp: Overhaul locations for _Pragma tokens

[Lewis Hyatt via Gcc-patches]

On Mon, Jul 31, 2023 at 06:39:15PM -0400, Lewis Hyatt wrote:
> On Fri, Jul 28, 2023 at 6:58 PM David Malcolm  wrote:
> >
> > On Fri, 2023-07-21 at 19:08 -0400, Lewis Hyatt wrote:
> > > Add a new linemap reason LC_GEN which enables encoding the location
> > > of data
> > > that was generated during compilation and does not appear in any
> > > source file.
> > > There could be many use cases, such as, for instance, referring to
> > > the content
> > > of builtin macros (not yet implemented, but an easy lift after this
> > > one.) The
> > > first intended application is to create a place to store the input to
> > > a
> > > _Pragma directive, so that proper locations can be assigned to those
> > > tokens. This will be done in a subsequent commit.
> > >
> > > The actual change needed to the line-maps API in libcpp is not too
> > > large and
> > > requires no space overhead in the line map data structures (on 64-bit
> > > systems
> > > that is; one newly added data member to class line_map_ordinary sits
> > > inside
> > > former padding bytes.) An LC_GEN map is just an ordinary map like any
> > > other,
> > > but the TO_FILE member that normally points to the file name points
> > > instead to
> > > the actual data.  This works automatically with PCH as well, for the
> > > same
> > > reason that the file name makes its way into a PCH.  In order to
> > > avoid
> > > confusion, the member has been renamed from TO_FILE to DATA, and
> > > associated
> > > accessors adjusted.
> > >
> > > Outside libcpp, there are many small changes but most of them are to
> > > selftests, which are necessarily more sensitive to implementation
> > > details. From the perspective of the user (the "user", here, being a
> > > frontend
> > > using line maps or else the diagnostics infrastructure), the chief
> > > visible
> > > change is that the function location_get_source_line() should be
> > > passed an
> > > expanded_location object instead of a separate filename and line
> > > number.  This
> > > is not a big change because in most cases, this information came
> > > anyway from a
> > > call to expand_location and the needed expanded_location object is
> > > readily
> > > available. The new overload of location_get_source_line() uses the
> > > extra
> > > information in the expanded_location object to obtain the data from
> > > the
> > > in-memory buffer when it originated from an LC_GEN map.
> > >
> > > Until the subsequent patch that starts using LC_GEN maps, none are
> > > yet
> > > generated within GCC, hence nothing is added to the testsuite here;
> > > but all
> > > relevant selftests have been extended to cover generated data maps in
> > > addition
> > > to normal files.
> >
> > [..snip...]
> >
> > Thanks for the updated patch.
> >
> > Reading this patch, it felt a bit unnatural to me to have an
> >   (exploded location, source line)
> > pair where the exploded location seems to be representing "which source
> > file or generated buffer", but the line/column info in that
> > exploded_location is to be ignored in favor of the 2nd source line.
> >
> > I think we're missing a class: something that identifies either a
> > specific source file, or a specific generated buffer.
> >
> > How about something like either:
> >
> > class source_id
> > {
> > public:
> >   source_id (const char *filename)
> >   : m_filename_or_buffer (filename),
> > m_len (0)
> >   {
> >   }
> >
> >   explicit source_id (const char *buffer, unsigned buffer_len)
> >   : m_filename_or_buffer (buffer),
> > m_len (buffer_len)
> >   {
> > linemap_assert (buffer_len > 0);
> >   }
> >
> > private:
> >   const char *m_filename_or_buffer;
> >   unsigned m_len;  // where 0 means "it's a filename"
> > };
> >
> > or:
> >
> > class source_id
> > {
> > public:
> >   source_id (const char *filename)
> >   : m_ptr (filename),
> > m_is_buffer (false)
> >   {
> >   }
> >
> >   explicit source_id (const linemap_ordinary *buffer_linemap)
> >   : m_ptr (buffer_linemap),
> > m_is_buffer (true)
> >   {
> >   }
> >
> > private:
> >   const void *m_ptr;
> > 

Re: [PATCH] preprocessor: c++: Support `#pragma GCC target' macros [PR87299]

[Lewis Hyatt via Gcc-patches]

On Tue, Aug 1, 2023 at 11:01 AM Joseph Myers  wrote:
>
> On Mon, 31 Jul 2023, Lewis Hyatt via Gcc-patches wrote:
>
> > I added some additional testcases from the PR for x86. The other targets
> > that support `#pragma GCC target' (aarch64, arm, nios2, powerpc, s390)
> > already had tests verifying that the pragma sets macros as expected; here I
> > have added -save-temps to some of them, to test that it now works in
> > preprocess-only mode as well.
>
> It would seem better to have copies of the tests with and without
> -save-temps, to test in both modes, rather than changing what's tested by
> an existing test here.  Or a test variant that #includes the original test
> but uses different options, if the original test isn't doing anything that
> would fail to work with that approach.

Thank you, I will adjust this.

-Lewis

[PATCH] preprocessor: c++: Support `#pragma GCC target' macros [PR87299]

[Lewis Hyatt via Gcc-patches]

`#pragma GCC target' is not currently handled in preprocess-only mode (e.g.,
when running gcc -E or gcc -save-temps). As noted in the PR, this means that
if the target pragma defines any macros, those macros are not effective in
preprocess-only mode. Similarly, such macros are not effective when
compiling with C++ (even when compiling without -save-temps), because C++
does not process the pragma until after all tokens have been obtained from
libcpp, at which point it is too late for macro expansion to take place.

Since r13-1544 and r14-2893, there is a general mechanism to handle pragmas
under these conditions as well, so resolve the PR by using the new "early
pragma" support.

toplev.cc required some changes because the target-specific handlers for
`#pragma GCC target' may call target_reinit(), and toplev.cc was not expecting
that function to be called in preprocess-only mode.

I added some additional testcases from the PR for x86. The other targets
that support `#pragma GCC target' (aarch64, arm, nios2, powerpc, s390)
already had tests verifying that the pragma sets macros as expected; here I
have added -save-temps to some of them, to test that it now works in
preprocess-only mode as well.

gcc/c-family/ChangeLog:

PR preprocessor/87299
* c-pragma.cc (init_pragma): Register `#pragma GCC target' and
related pragmas in preprocess-only mode, and enable early handling.
(c_reset_target_pragmas): New function refactoring code from...
(handle_pragma_reset_options): ...here.
* c-pragma.h (c_reset_target_pragmas): Declare.

gcc/cp/ChangeLog:

PR preprocessor/87299
* parser.cc (cp_lexer_new_main): Call c_reset_target_pragmas ()
after preprocessing is complete, before starting compilation.

gcc/ChangeLog:

PR preprocessor/87299
* toplev.cc (no_backend): New static global.
(finalize): Remove argument no_backend, which is now a
static global.
(process_options): Likewise.
(do_compile): Likewise.
(target_reinit): Don't do anything in preprocess-only mode.
(toplev::main): Adapt to no_backend change.
(toplev::finalize): Likewise.

gcc/testsuite/ChangeLog:

PR preprocessor/87299
* c-c++-common/pragma-target-1.c: New test.
* c-c++-common/pragma-target-2.c: New test.
* g++.target/i386/pr87299-1.C: New test.
* g++.target/i386/pr87299-2.C: New test.
* gcc.target/i386/pr87299-1.c: New test.
* gcc.target/i386/pr87299-2.c: New test.
* gcc.target/s390/target-attribute/tattr-2.c: Add -save-temps to the
options, to test preprocess-only mode as well.
* gcc.target/aarch64/pragma_cpp_predefs_1.c: Likewise.
* gcc.target/arm/pragma_arch_attribute.c: Likewise.
* gcc.target/nios2/custom-fp-2.c: Likewise.
* gcc.target/powerpc/float128-3.c: Likewise.
---

Notes:
Hello-

This patch fixes the PR by enabling early pragma handling for `#pragma GCC
target' and related pragmas such as `#pragma GCC push_options'. I did not
need to touch any target-specific code, however I did need to make a change
to toplev.cc, affecting all targets, to make it safe to call target_reinit()
in preprocess-only mode. (Otherwise, it would be necessary to modify the
implementation of target pragmas in every target, to avoid this code path.)
That was the only complication I ran into.

Regarding testing, I did: (thanks to GCC compile farm for the non-x86
targets)

bootstrap + regtest all languages - x86_64-pc-linux-gnu
bootstrap + regtest c/c++ - powerpc64le-unknown-linux-gnu,
aarch64-unknown-linux-gnu

The following backends also implement this pragma so ought to be tested:
arm
nios2
s390

I am not able to test those directly. I did add coverage to their testsuites
(basically, adding -save-temps to any existing test, causes it to test the
pragma in preprocess-only mode.) Then, I verified on x86_64 with a cross
compiler, that the modified testcases fail before the patch and pass
afterwards. nios2 is an exception, it does not set any libcpp macros when
handling the pragma, so there is nothing to test, but I did verify that
processing the pragma in preprocess-only mode does not cause any problems.
The cross compilers tested were targets arm-unknown-linux-gnueabi,
nios2-unknown-linux, and s390-ibm-linux.

Please let me know if it looks OK? Thanks!

-Lewis

 gcc/c-family/c-pragma.cc  | 49 ---
 gcc/c-family/c-pragma.h   |  2 +-
 gcc/cp/parser.cc  |  6 +++
 gcc/testsuite/c-c++-common/pragma-target-1.c  | 19 +++
 gcc/testsuite/c-c++-common/pragma-target-2.c  | 27 ++
 gcc/testsuite/g++.target/i386/pr87299-1.C |  8 +++
 gcc/testsuite/g++.target/i386/pr87299-2.C

Re: [PATCH v3 1/4] diagnostics: libcpp: Add LC_GEN linemaps to support in-memory buffers

[Lewis Hyatt via Gcc-patches]

On Fri, Jul 28, 2023 at 6:58 PM David Malcolm  wrote:
>
> On Fri, 2023-07-21 at 19:08 -0400, Lewis Hyatt wrote:
> > Add a new linemap reason LC_GEN which enables encoding the location
> > of data
> > that was generated during compilation and does not appear in any
> > source file.
> > There could be many use cases, such as, for instance, referring to
> > the content
> > of builtin macros (not yet implemented, but an easy lift after this
> > one.) The
> > first intended application is to create a place to store the input to
> > a
> > _Pragma directive, so that proper locations can be assigned to those
> > tokens. This will be done in a subsequent commit.
> >
> > The actual change needed to the line-maps API in libcpp is not too
> > large and
> > requires no space overhead in the line map data structures (on 64-bit
> > systems
> > that is; one newly added data member to class line_map_ordinary sits
> > inside
> > former padding bytes.) An LC_GEN map is just an ordinary map like any
> > other,
> > but the TO_FILE member that normally points to the file name points
> > instead to
> > the actual data.  This works automatically with PCH as well, for the
> > same
> > reason that the file name makes its way into a PCH.  In order to
> > avoid
> > confusion, the member has been renamed from TO_FILE to DATA, and
> > associated
> > accessors adjusted.
> >
> > Outside libcpp, there are many small changes but most of them are to
> > selftests, which are necessarily more sensitive to implementation
> > details. From the perspective of the user (the "user", here, being a
> > frontend
> > using line maps or else the diagnostics infrastructure), the chief
> > visible
> > change is that the function location_get_source_line() should be
> > passed an
> > expanded_location object instead of a separate filename and line
> > number.  This
> > is not a big change because in most cases, this information came
> > anyway from a
> > call to expand_location and the needed expanded_location object is
> > readily
> > available. The new overload of location_get_source_line() uses the
> > extra
> > information in the expanded_location object to obtain the data from
> > the
> > in-memory buffer when it originated from an LC_GEN map.
> >
> > Until the subsequent patch that starts using LC_GEN maps, none are
> > yet
> > generated within GCC, hence nothing is added to the testsuite here;
> > but all
> > relevant selftests have been extended to cover generated data maps in
> > addition
> > to normal files.
>
> [..snip...]
>
> Thanks for the updated patch.
>
> Reading this patch, it felt a bit unnatural to me to have an
>   (exploded location, source line)
> pair where the exploded location seems to be representing "which source
> file or generated buffer", but the line/column info in that
> exploded_location is to be ignored in favor of the 2nd source line.
>
> I think we're missing a class: something that identifies either a
> specific source file, or a specific generated buffer.
>
> How about something like either:
>
> class source_id
> {
> public:
>   source_id (const char *filename)
>   : m_filename_or_buffer (filename),
> m_len (0)
>   {
>   }
>
>   explicit source_id (const char *buffer, unsigned buffer_len)
>   : m_filename_or_buffer (buffer),
> m_len (buffer_len)
>   {
> linemap_assert (buffer_len > 0);
>   }
>
> private:
>   const char *m_filename_or_buffer;
>   unsigned m_len;  // where 0 means "it's a filename"
> };
>
> or:
>
> class source_id
> {
> public:
>   source_id (const char *filename)
>   : m_ptr (filename),
> m_is_buffer (false)
>   {
>   }
>
>   explicit source_id (const linemap_ordinary *buffer_linemap)
>   : m_ptr (buffer_linemap),
> m_is_buffer (true)
>   {
>   }
>
> private:
>   const void *m_ptr;
>   bool m_is_buffer;
> };
>
> and use one of these "source_id file" in place of "const char *file",
> rather than replacing such things with expanded_location?
>
> > diff --git a/gcc/c-family/c-indentation.cc b/gcc/c-family/c-indentation.cc
> > index e8d3dece770..4164fa0b1ba 100644
> > --- a/gcc/c-family/c-indentation.cc
> > +++ b/gcc/c-family/c-indentation.cc
> > @@ -50,7 +50,7 @@ get_visual_column (expanded_location exploc,
> >  unsigned int *first_nws,
> >  unsigned int tab_width)
> >  {
> > -  char_span line = location_get_

Re: [PATCH v3 0/4] diagnostics: libcpp: Overhaul locations for _Pragma tokens

[Lewis Hyatt via Gcc-patches]

On Fri, Jul 28, 2023 at 6:22 PM David Malcolm  wrote:
>
> On Fri, 2023-07-21 at 19:08 -0400, Lewis Hyatt wrote:
> > Hello-
> >
> > This is an update to the v2 patch series last sent in January:
> > https://gcc.gnu.org/pipermail/gcc-patches/2023-January/609473.html
> >
> > While I did not receive any feedback on the v2 patches yet, they did
> > need some
> > rebasing on top of other recent commits to input.cc, so I thought it
> > would be
> > helpful to send them again now. The patches have not otherwise
> > changed from
> > v2, and the above-linked message explains how all the patches fit in
> > with the
> > original v1 series sent last November.
> >
> > Dave, I would appreciate it very much if you could please let me know
> > what you
> > think of this approach? I feel like the diagnostics we currently
> > output for _Pragmas are worth improving. As a reminder, say for this
> > example:
> >
> > =
> >  #define S "GCC diagnostic ignored \"oops"
> >  _Pragma(S)
> > =
> >
> > We currently output:
> >
> > =
> > file.cpp:2:24: warning: missing terminating " character
> > 2 | _Pragma(S)
> >   |^
> > =
> >
> > While after these patches, we would output:
> >
> > ==
> > :1:24: warning: missing terminating " character
> > 1 | GCC diagnostic ignored "oops
> >   |^
> > file.cpp:2:1: note: in <_Pragma directive>
> > 2 | _Pragma(S)
> >   | ^~~
> > ==
> >
> > Thanks!
>
> Hi Lewis; sorry for not responding to the v2 patches.
>
> I've started looking at the v3 patches in detail, but I have some high-
> level questions about memory usage:
>
> Am I right in thinking that the effect of this patch is that for every
> _Pragma in the source we will create a new line_map_ordinary, and a new
> buffer for the stringified content of that _Pragma, and that these
> allocations will persist for the rest of the compilation?  (plus a
> little extra allocation within the "location_t" space from 0 to
> 0x7fff).
>
> It sounds like this will probably be a rounding error that won't be
> noticable in profiling, but did you attempt any such measurement of the
> memory usage before/after this patch on some real-world projects?
>
> Thanks
> Dave
>

Thanks for looking at the patches, I appreciate it whenever you have
time to get to them.

This is a fair point about the memory usage, basically it means that
each instance of a _Pragma has comparable memory footprint to a macro
definition. (In addition to the overheads you mentioned, it also
creates a macro map to generate a virtual location for the tokens, so
that it's able to output the "in expansion of _Pragma" note. That part
can be disabled with -ftrack-macro-expansion=0 at least.)

I had the sense that _Pragma isn't used often enough for that to be a
problem, but agreed it is worth checking. (I really hope this memory
usage isn't an issue since there are also numerous PRs complaining
about 32-bit limitations in location tracking, that make it tempting
to explore 64-bit line maps or some other option someday too.)

I tried one thing now, wxWidgets uses a lot of diagnostic pragmas
wrapped up inside macros that use _Pragma. (See
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578). The testsuite
contains a file allheaders.cpp which includes the whole library, so I
tried compiling this into a pch, which I believe measures the entire
memory footprint including the ordinary and macro line maps and the
_Pragma strings. The resulting PCH sizes were:

279000173 bytes before the changes
279491345 bytes after the changes

So 0.1% bigger. Happy to check other projects too, do you have any
standard gotos? Maybe firefox or something I take it.

I see your other response on patch #1, I am thinking about that and
will reply later. Thanks again!

-Lewis

Re: [PATCH v2] c-family: Implement pragma_lex () for preprocess-only mode

[Lewis Hyatt via Gcc-patches]

On Thu, Jul 27, 2023 at 06:18:33PM -0700, Jason Merrill wrote:
> On 7/27/23 18:59, Lewis Hyatt wrote:
> > In order to support processing #pragma in preprocess-only mode (-E or
> > -save-temps for gcc/g++), we need a way to obtain the #pragma tokens from
> > libcpp. In full compilation modes, this is accomplished by calling
> > pragma_lex (), which is a symbol that must be exported by the frontend, and
> > which is currently implemented for C and C++. Neither of those frontends
> > initializes its parser machinery in preprocess-only mode, and consequently
> > pragma_lex () does not work in this case.
> > 
> > Address that by adding a new function c_init_preprocess () for the frontends
> > to implement, which arranges for pragma_lex () to work in preprocess-only
> > mode, and adjusting pragma_lex () accordingly.
> > 
> > In preprocess-only mode, the preprocessor is accustomed to controlling the
> > interaction with libcpp, and it only knows about tokens that it has called
> > into libcpp itself to obtain. Since it still needs to see the tokens
> > obtained by pragma_lex () so that they can be streamed to the output, also
> > adjust c_lex_with_flags () and related functions in c-family/c-lex.cc to
> > inform the preprocessor about any tokens it won't be aware of.
> > 
> > Currently, there is one place where we are already supporting #pragma in
> > preprocess-only mode, namely the handling of `#pragma GCC diagnostic'.  That
> > was done by directly interfacing with libcpp, rather than making use of
> > pragma_lex (). Now that pragma_lex () works, that code is no longer
> > necessary; remove it.
> > 
> > gcc/c-family/ChangeLog:
> > 
> > * c-common.h (c_init_preprocess): Declare.
> > (c_lex_enable_token_streaming): Declare.
> > * c-opts.cc (c_common_init): Call c_init_preprocess ().
> > * c-lex.cc (stream_tokens_to_preprocessor): New static variable.
> > (c_lex_enable_token_streaming): New function.
> > (cb_def_pragma): Add a comment.
> > (get_token): New function wrapping cpp_get_token.
> > (c_lex_with_flags): Use the new wrapper function to support
> > obtaining tokens in preprocess_only mode.
> > (lex_string): Likewise.
> > * c-ppoutput.cc (preprocess_file): Call c_lex_enable_token_streaming
> > when needed.
> > * c-pragma.cc (pragma_diagnostic_lex_normal): Rename to...
> > (pragma_diagnostic_lex): ...this.
> > (pragma_diagnostic_lex_pp): Remove.
> > (handle_pragma_diagnostic_impl): Call pragma_diagnostic_lex () in
> > all modes.
> > (c_pp_invoke_early_pragma_handler): Adapt to support pragma_lex ()
> > usage.
> > * c-pragma.h (pragma_lex_discard_to_eol): Declare.
> > 
> > gcc/c/ChangeLog:
> > 
> > * c-parser.cc (pragma_lex_discard_to_eol): New function.
> > (c_init_preprocess): New function.
> > 
> > gcc/cp/ChangeLog:
> > 
> > * parser.cc (c_init_preprocess): New function.
> > (maybe_read_tokens_for_pragma_lex): New function.
> > (pragma_lex): Support preprocess-only mode.
> > (pragma_lex_discard_to_eol): New function.
> > ---
> > 
> > Notes:
> >  Hello-
> >  Here is version 2 of the patch, incorporating Jason's feedback from
> >  https://gcc.gnu.org/pipermail/gcc-patches/2023-July/625591.html
> >  Thanks again, please let me know if it's OK? Bootstrap + regtest all
> >  languages on x86-64 Linux looks good.
> >  -Lewis
> > 
> >   gcc/c-family/c-common.h|  4 +++
> >   gcc/c-family/c-lex.cc  | 49 +
> >   gcc/c-family/c-opts.cc |  1 +
> >   gcc/c-family/c-ppoutput.cc | 17 +---
> >   gcc/c-family/c-pragma.cc   | 56 ++
> >   gcc/c-family/c-pragma.h|  2 ++
> >   gcc/c/c-parser.cc  | 21 ++
> >   gcc/cp/parser.cc   | 45 ++
> >   8 files changed, 138 insertions(+), 57 deletions(-)
> > 
> > diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
> > index b5ef5ff6b2c..2fe2f194660 100644
> > --- a/gcc/c-family/c-common.h
> > +++ b/gcc/c-family/c-common.h
> > @@ -990,6 +990,9 @@ extern void c_parse_file (void);
> >   extern void c_parse_final_cleanups (void);
> > +/* This initializes for preprocess-only mode.  */
> > +extern void c_init_preprocess (void);
> > +
> >   /* These macros provide convenient access to the various _STMT nodes.  */
> >   /* Nonzero if a given STATEMENT_LIST repre

[PATCH v2] c-family: Implement pragma_lex () for preprocess-only mode

[Lewis Hyatt via Gcc-patches]

In order to support processing #pragma in preprocess-only mode (-E or
-save-temps for gcc/g++), we need a way to obtain the #pragma tokens from
libcpp. In full compilation modes, this is accomplished by calling
pragma_lex (), which is a symbol that must be exported by the frontend, and
which is currently implemented for C and C++. Neither of those frontends
initializes its parser machinery in preprocess-only mode, and consequently
pragma_lex () does not work in this case.

Address that by adding a new function c_init_preprocess () for the frontends
to implement, which arranges for pragma_lex () to work in preprocess-only
mode, and adjusting pragma_lex () accordingly.

In preprocess-only mode, the preprocessor is accustomed to controlling the
interaction with libcpp, and it only knows about tokens that it has called
into libcpp itself to obtain. Since it still needs to see the tokens
obtained by pragma_lex () so that they can be streamed to the output, also
adjust c_lex_with_flags () and related functions in c-family/c-lex.cc to
inform the preprocessor about any tokens it won't be aware of.

Currently, there is one place where we are already supporting #pragma in
preprocess-only mode, namely the handling of `#pragma GCC diagnostic'.  That
was done by directly interfacing with libcpp, rather than making use of
pragma_lex (). Now that pragma_lex () works, that code is no longer
necessary; remove it.

gcc/c-family/ChangeLog:

* c-common.h (c_init_preprocess): Declare.
(c_lex_enable_token_streaming): Declare.
* c-opts.cc (c_common_init): Call c_init_preprocess ().
* c-lex.cc (stream_tokens_to_preprocessor): New static variable.
(c_lex_enable_token_streaming): New function.
(cb_def_pragma): Add a comment.
(get_token): New function wrapping cpp_get_token.
(c_lex_with_flags): Use the new wrapper function to support
obtaining tokens in preprocess_only mode.
(lex_string): Likewise.
* c-ppoutput.cc (preprocess_file): Call c_lex_enable_token_streaming
when needed.
* c-pragma.cc (pragma_diagnostic_lex_normal): Rename to...
(pragma_diagnostic_lex): ...this.
(pragma_diagnostic_lex_pp): Remove.
(handle_pragma_diagnostic_impl): Call pragma_diagnostic_lex () in
all modes.
(c_pp_invoke_early_pragma_handler): Adapt to support pragma_lex ()
usage.
* c-pragma.h (pragma_lex_discard_to_eol): Declare.

gcc/c/ChangeLog:

* c-parser.cc (pragma_lex_discard_to_eol): New function.
(c_init_preprocess): New function.

gcc/cp/ChangeLog:

* parser.cc (c_init_preprocess): New function.
(maybe_read_tokens_for_pragma_lex): New function.
(pragma_lex): Support preprocess-only mode.
(pragma_lex_discard_to_eol): New function.
---

Notes:
Hello-

Here is version 2 of the patch, incorporating Jason's feedback from
https://gcc.gnu.org/pipermail/gcc-patches/2023-July/625591.html

Thanks again, please let me know if it's OK? Bootstrap + regtest all
languages on x86-64 Linux looks good.

-Lewis

 gcc/c-family/c-common.h|  4 +++
 gcc/c-family/c-lex.cc  | 49 +
 gcc/c-family/c-opts.cc |  1 +
 gcc/c-family/c-ppoutput.cc | 17 +---
 gcc/c-family/c-pragma.cc   | 56 ++
 gcc/c-family/c-pragma.h|  2 ++
 gcc/c/c-parser.cc  | 21 ++
 gcc/cp/parser.cc   | 45 ++
 8 files changed, 138 insertions(+), 57 deletions(-)

diff --git a/gcc/c-family/c-common.h b/gcc/c-family/c-common.h
index b5ef5ff6b2c..2fe2f194660 100644
--- a/gcc/c-family/c-common.h
+++ b/gcc/c-family/c-common.h
@@ -990,6 +990,9 @@ extern void c_parse_file (void);
 
 extern void c_parse_final_cleanups (void);
 
+/* This initializes for preprocess-only mode.  */
+extern void c_init_preprocess (void);
+
 /* These macros provide convenient access to the various _STMT nodes.  */
 
 /* Nonzero if a given STATEMENT_LIST represents the outermost binding
@@ -1214,6 +1217,7 @@ extern tree c_build_bind_expr (location_t, tree, tree);
 /* In c-lex.cc.  */
 extern enum cpp_ttype
 conflict_marker_get_final_tok_kind (enum cpp_ttype tok1_kind);
+extern void c_lex_enable_token_streaming (bool enabled);
 
 /* In c-pch.cc  */
 extern void pch_init (void);
diff --git a/gcc/c-family/c-lex.cc b/gcc/c-family/c-lex.cc
index dcd061c7cb1..ac4c018d863 100644
--- a/gcc/c-family/c-lex.cc
+++ b/gcc/c-family/c-lex.cc
@@ -57,6 +57,17 @@ static void cb_ident (cpp_reader *, unsigned int, const 
cpp_string *);
 static void cb_def_pragma (cpp_reader *, unsigned int);
 static void cb_define (cpp_reader *, unsigned int, cpp_hashnode *);
 static void cb_undef (cpp_reader *, unsigned int, cpp_hashnode *);
+
+/* Flag to remember if we are in a mode (such as flag_preprocess_only) in which
+   tokens obtained here need to be streamed to the preprocessor.  */
+stat