[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-10-09 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw updated this revision to Diff 378476.
woodruffw added a comment.

Resolve conflicts.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

Files:
  clang/lib/Frontend/ASTConsumers.cpp
  clang/test/AST/ast-dump-comment-json.cpp
  clang/test/AST/ast-dump-decl-context-json.cpp
  clang/test/AST/ast-dump-decl-json.c
  clang/test/AST/ast-dump-decl-json.m
  clang/test/AST/ast-dump-enum-json.cpp
  clang/test/AST/ast-dump-expr-json.c
  clang/test/AST/ast-dump-expr-json.cpp
  clang/test/AST/ast-dump-expr-json.m
  clang/test/AST/ast-dump-file-line-json.c
  clang/test/AST/ast-dump-funcs-json.cpp
  clang/test/AST/ast-dump-if-json.cpp
  clang/test/AST/ast-dump-macro-json.c
  clang/test/AST/ast-dump-namespace-json.cpp
  clang/test/AST/ast-dump-record-definition-data-json.cpp
  clang/test/AST/ast-dump-records-json.cpp
  clang/test/AST/ast-dump-stmt-json.c
  clang/test/AST/ast-dump-stmt-json.cpp
  clang/test/AST/ast-dump-stmt-json.m
  clang/test/AST/ast-dump-template-decls-json.cpp
  clang/test/AST/ast-dump-temporaries-json.cpp
  clang/test/AST/ast-dump-types-json.cpp
  clang/test/AST/gen_ast_dump_json_test.py

Index: clang/test/AST/gen_ast_dump_json_test.py
===
--- clang/test/AST/gen_ast_dump_json_test.py
+++ clang/test/AST/gen_ast_dump_json_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 from collections import OrderedDict
@@ -6,7 +6,6 @@
 import argparse
 import json
 import os
-import pprint
 import re
 import subprocess
 import sys
@@ -21,20 +20,19 @@
 for e in v:
 if isinstance(e, OrderedDict):
 normalize(e)
-elif type(v) is unicode:
-st = v.encode('utf-8')
+elif type(v) is str:
 if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
-dict_var[k] = u'0x{{.*}}'
+dict_var[k] = '0x{{.*}}'
 elif os.path.isfile(v):
-dict_var[k] = u'{{.*}}'
+dict_var[k] = '{{.*}}'
 else:
-splits = (v.split(u' '))
+splits = (v.split(' '))
 out_splits = []
 for split in splits:
-inner_splits = split.rsplit(u':',2)
+inner_splits = split.rsplit(':',2)
 if os.path.isfile(inner_splits[0]):
 out_splits.append(
-u'{{.*}}:%s:%s'
+'{{.*}}:%s:%s'
 %(inner_splits[1],
   inner_splits[2]))
 continue
@@ -42,11 +40,11 @@
 
 dict_var[k] = ' '.join(out_splits)
 
+
 def filter_json(dict_var, filters, out):
 for k, v in dict_var.items():
-if type(v) is unicode:
-st = v.encode('utf-8')
-if st in filters:
+if type(v) is str:
+if v in filters:
 out.append(dict_var)
 break
 elif isinstance(v, OrderedDict):
@@ -154,33 +152,39 @@
 print("Will use the following filters:", filters)
 
 try:
-json_str = subprocess.check_output(cmd)
+json_str = subprocess.check_output(cmd).decode()
 except Exception as ex:
 print("The clang command failed with %s" % ex)
 return -1
 
 out_asts = []
 if using_ast_dump_filter:
-splits = re.split('Dumping .*:\n', json_str)
-if len(splits) > 1:
-for split in splits[1:]:
-j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict)
+# If we're using a filter, then we might have multiple JSON objects
+# in the output. To parse each out, we use a manual JSONDecoder in
+# "raw" mode and update our location in the string based on where the
+# last document ended.
+decoder = json.JSONDecoder(object_hook=OrderedDict)
+doc_start = 0
+prev_end = 0
+while True:
+try:
+prev_end = doc_start
+(j, doc_start) = decoder.raw_decode(json_str[doc_start:])
+doc_start += prev_end + 1
 normalize(j)
 out_asts.append(j)
+except:
+break
 else:
-j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict)
+j = json.loads(json_str, object_pairs_hook=OrderedDict)
 normalize(j)
 
 if len(filters) == 0:
 out_asts.append(j)
 else:
-#assert using_ast_dump_filter is False,\
-#"Does not support using compiler's ast-dump-filter "\
-#"and the tool's filter option at the same time yet."
-
 filter_json(j, filters, out_asts)
 
-with tempfile.NamedTemporaryFile("wb", delete=F

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-09-29 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw added a comment.

Another gentle ping for review.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-09-14 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw updated this revision to Diff 372595.
woodruffw added a comment.

Increase the context for the patch.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

Files:
  clang/lib/Frontend/ASTConsumers.cpp
  clang/test/AST/ast-dump-comment-json.cpp
  clang/test/AST/ast-dump-decl-context-json.cpp
  clang/test/AST/ast-dump-decl-json.c
  clang/test/AST/ast-dump-decl-json.m
  clang/test/AST/ast-dump-enum-json.cpp
  clang/test/AST/ast-dump-expr-json.c
  clang/test/AST/ast-dump-expr-json.cpp
  clang/test/AST/ast-dump-expr-json.m
  clang/test/AST/ast-dump-file-line-json.c
  clang/test/AST/ast-dump-funcs-json.cpp
  clang/test/AST/ast-dump-if-json.cpp
  clang/test/AST/ast-dump-macro-json.c
  clang/test/AST/ast-dump-namespace-json.cpp
  clang/test/AST/ast-dump-record-definition-data-json.cpp
  clang/test/AST/ast-dump-records-json.cpp
  clang/test/AST/ast-dump-stmt-json.c
  clang/test/AST/ast-dump-stmt-json.cpp
  clang/test/AST/ast-dump-stmt-json.m
  clang/test/AST/ast-dump-template-decls-json.cpp
  clang/test/AST/ast-dump-temporaries-json.cpp
  clang/test/AST/ast-dump-types-json.cpp
  clang/test/AST/gen_ast_dump_json_test.py

Index: clang/test/AST/gen_ast_dump_json_test.py
===
--- clang/test/AST/gen_ast_dump_json_test.py
+++ clang/test/AST/gen_ast_dump_json_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 from collections import OrderedDict
@@ -6,7 +6,6 @@
 import argparse
 import json
 import os
-import pprint
 import re
 import subprocess
 import sys
@@ -21,20 +20,19 @@
 for e in v:
 if isinstance(e, OrderedDict):
 normalize(e)
-elif type(v) is unicode:
-st = v.encode('utf-8')
+elif type(v) is str:
 if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
-dict_var[k] = u'0x{{.*}}'
+dict_var[k] = '0x{{.*}}'
 elif os.path.isfile(v):
-dict_var[k] = u'{{.*}}'
+dict_var[k] = '{{.*}}'
 else:
-splits = (v.split(u' '))
+splits = (v.split(' '))
 out_splits = []
 for split in splits:
-inner_splits = split.rsplit(u':',2)
+inner_splits = split.rsplit(':',2)
 if os.path.isfile(inner_splits[0]):
 out_splits.append(
-u'{{.*}}:%s:%s'
+'{{.*}}:%s:%s'
 %(inner_splits[1],
   inner_splits[2]))
 continue
@@ -42,11 +40,11 @@
 
 dict_var[k] = ' '.join(out_splits)
 
+
 def filter_json(dict_var, filters, out):
 for k, v in dict_var.items():
-if type(v) is unicode:
-st = v.encode('utf-8')
-if st in filters:
+if type(v) is str:
+if v in filters:
 out.append(dict_var)
 break
 elif isinstance(v, OrderedDict):
@@ -154,33 +152,39 @@
 print("Will use the following filters:", filters)
 
 try:
-json_str = subprocess.check_output(cmd)
+json_str = subprocess.check_output(cmd).decode()
 except Exception as ex:
 print("The clang command failed with %s" % ex)
 return -1
 
 out_asts = []
 if using_ast_dump_filter:
-splits = re.split('Dumping .*:\n', json_str)
-if len(splits) > 1:
-for split in splits[1:]:
-j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict)
+# If we're using a filter, then we might have multiple JSON objects
+# in the output. To parse each out, we use a manual JSONDecoder in
+# "raw" mode and update our location in the string based on where the
+# last document ended.
+decoder = json.JSONDecoder(object_hook=OrderedDict)
+doc_start = 0
+prev_end = 0
+while True:
+try:
+prev_end = doc_start
+(j, doc_start) = decoder.raw_decode(json_str[doc_start:])
+doc_start += prev_end + 1
 normalize(j)
 out_asts.append(j)
+except:
+break
 else:
-j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict)
+j = json.loads(json_str, object_pairs_hook=OrderedDict)
 normalize(j)
 
 if len(filters) == 0:
 out_asts.append(j)
 else:
-#assert using_ast_dump_filter is False,\
-#"Does not support using compiler's ast-dump-filter "\
-#"and the tool's filter option at the same time yet."
-
 filter_json(j, filters, out_asts)
 
-with tempfile.NamedTemporaryFi

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-09-08 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw added a comment.

Another gentle ping for review.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-08-30 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw added a comment.

Gentle ping for review on this!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-08-20 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw updated this revision to Diff 367938.
woodruffw added a comment.

Updated the AST dump tests for JSON to refute the presence of the `Dumping ` 
prefix.

Also, updated the `gen_ast_dump_json_test.py` generator to work with Python 3 
and to behave better without the presence of the `Dumping: ` sentinels.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D108441/new/

https://reviews.llvm.org/D108441

Files:
  clang/lib/Frontend/ASTConsumers.cpp
  clang/test/AST/ast-dump-comment-json.cpp
  clang/test/AST/ast-dump-decl-context-json.cpp
  clang/test/AST/ast-dump-decl-json.c
  clang/test/AST/ast-dump-decl-json.m
  clang/test/AST/ast-dump-enum-json.cpp
  clang/test/AST/ast-dump-expr-json.c
  clang/test/AST/ast-dump-expr-json.cpp
  clang/test/AST/ast-dump-expr-json.m
  clang/test/AST/ast-dump-file-line-json.c
  clang/test/AST/ast-dump-funcs-json.cpp
  clang/test/AST/ast-dump-if-json.cpp
  clang/test/AST/ast-dump-macro-json.c
  clang/test/AST/ast-dump-namespace-json.cpp
  clang/test/AST/ast-dump-record-definition-data-json.cpp
  clang/test/AST/ast-dump-records-json.cpp
  clang/test/AST/ast-dump-stmt-json.c
  clang/test/AST/ast-dump-stmt-json.cpp
  clang/test/AST/ast-dump-stmt-json.m
  clang/test/AST/ast-dump-template-decls-json.cpp
  clang/test/AST/ast-dump-temporaries-json.cpp
  clang/test/AST/ast-dump-types-json.cpp
  clang/test/AST/gen_ast_dump_json_test.py

Index: clang/test/AST/gen_ast_dump_json_test.py
===
--- clang/test/AST/gen_ast_dump_json_test.py
+++ clang/test/AST/gen_ast_dump_json_test.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 from collections import OrderedDict
@@ -6,7 +6,6 @@
 import argparse
 import json
 import os
-import pprint
 import re
 import subprocess
 import sys
@@ -21,20 +20,19 @@
 for e in v:
 if isinstance(e, OrderedDict):
 normalize(e)
-elif type(v) is unicode:
-st = v.encode('utf-8')
+elif type(v) is str:
 if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
-dict_var[k] = u'0x{{.*}}'
+dict_var[k] = '0x{{.*}}'
 elif os.path.isfile(v):
-dict_var[k] = u'{{.*}}'
+dict_var[k] = '{{.*}}'
 else:
-splits = (v.split(u' '))
+splits = (v.split(' '))
 out_splits = []
 for split in splits:
-inner_splits = split.rsplit(u':',2)
+inner_splits = split.rsplit(':',2)
 if os.path.isfile(inner_splits[0]):
 out_splits.append(
-u'{{.*}}:%s:%s'
+'{{.*}}:%s:%s'
 %(inner_splits[1],
   inner_splits[2]))
 continue
@@ -42,11 +40,11 @@
 
 dict_var[k] = ' '.join(out_splits)
 
+
 def filter_json(dict_var, filters, out):
 for k, v in dict_var.items():
-if type(v) is unicode:
-st = v.encode('utf-8')
-if st in filters:
+if type(v) is str:
+if v in filters:
 out.append(dict_var)
 break
 elif isinstance(v, OrderedDict):
@@ -154,33 +152,39 @@
 print("Will use the following filters:", filters)
 
 try:
-json_str = subprocess.check_output(cmd)
+json_str = subprocess.check_output(cmd).decode()
 except Exception as ex:
 print("The clang command failed with %s" % ex)
 return -1
 
 out_asts = []
 if using_ast_dump_filter:
-splits = re.split('Dumping .*:\n', json_str)
-if len(splits) > 1:
-for split in splits[1:]:
-j = json.loads(split.decode('utf-8'), object_pairs_hook=OrderedDict)
+# If we're using a filter, then we might have multiple JSON objects
+# in the output. To parse each out, we use a manual JSONDecoder in
+# "raw" mode and update our location in the string based on where the
+# last document ended.
+decoder = json.JSONDecoder(object_hook=OrderedDict)
+doc_start = 0
+prev_end = 0
+while True:
+try:
+prev_end = doc_start
+(j, doc_start) = decoder.raw_decode(json_str[doc_start:])
+doc_start += prev_end + 1
 normalize(j)
 out_asts.append(j)
+except:
+break
 else:
-j = json.loads(json_str.decode('utf-8'), object_pairs_hook=OrderedDict)
+j = json.loads(json_str, object_pairs_hook=OrderedDict)
 normalize(j)
 
 if len(filters) == 0:
 out_asts.append(j)
 else:
-#assert using_ast_dump_filter is False,\
-#"Does not support using c

[PATCH] D108441: [clang] Fix JSON AST output when a filter is used

2021-08-19 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw created this revision.
woodruffw added reviewers: klimek, rsmith.
woodruffw added a project: clang.
woodruffw requested review of this revision.
Herald added a subscriber: cfe-commits.

Without this, the combination of `-ast-dump=json` and `-ast-dump-filter FILTER` 
produces invalid JSON: the first line is a string that says `Dumping 
$SOME_DECL_NAME: `.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D108441

Files:
  clang/lib/Frontend/ASTConsumers.cpp


Index: clang/lib/Frontend/ASTConsumers.cpp
===
--- clang/lib/Frontend/ASTConsumers.cpp
+++ clang/lib/Frontend/ASTConsumers.cpp
@@ -57,8 +57,11 @@
 bool ShowColors = Out.has_colors();
 if (ShowColors)
   Out.changeColor(raw_ostream::BLUE);
-Out << (OutputKind != Print ? "Dumping " : "Printing ") << getName(D)
-<< ":\n";
+
+if (OutputFormat == ADOF_Default)
+  Out << (OutputKind != Print ? "Dumping " : "Printing ") << getName(D)
+  << ":\n";
+
 if (ShowColors)
   Out.resetColor();
 print(D);


Index: clang/lib/Frontend/ASTConsumers.cpp
===
--- clang/lib/Frontend/ASTConsumers.cpp
+++ clang/lib/Frontend/ASTConsumers.cpp
@@ -57,8 +57,11 @@
 bool ShowColors = Out.has_colors();
 if (ShowColors)
   Out.changeColor(raw_ostream::BLUE);
-Out << (OutputKind != Print ? "Dumping " : "Printing ") << getName(D)
-<< ":\n";
+
+if (OutputFormat == ADOF_Default)
+  Out << (OutputKind != Print ? "Dumping " : "Printing ") << getName(D)
+  << ":\n";
+
 if (ShowColors)
   Out.resetColor();
 print(D);
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63697: clang-format: Fix error return when using inplace with stdin

2019-06-28 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw added a comment.

Yep, I'll add one in a bit.


Repository:
  rC Clang

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D63697/new/

https://reviews.llvm.org/D63697



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D63697: clang-format: Fix error return when using inplace with stdin

2019-06-23 Thread William Woodruff via Phabricator via cfe-commits 
woodruffw created this revision.
woodruffw added reviewers: clang, thakis.
woodruffw added a project: clang.
Herald added a subscriber: cfe-commits.

Causes clang-format to exit with an appropriate error code
when the invalid combination of -i and stdin is provided.


Repository:
  rC Clang

https://reviews.llvm.org/D63697

Files:
  tools/clang-format/ClangFormat.cpp


Index: tools/clang-format/ClangFormat.cpp
===
--- tools/clang-format/ClangFormat.cpp
+++ tools/clang-format/ClangFormat.cpp
@@ -243,7 +243,7 @@
 static bool format(StringRef FileName) {
   if (!OutputXML && Inplace && FileName == "-") {
 errs() << "error: cannot use -i when reading from stdin.\n";
-return false;
+return true;
   }
   // On Windows, overwriting a file with an open file mapping doesn't work,
   // so read the whole file into memory when formatting in-place.


Index: tools/clang-format/ClangFormat.cpp
===
--- tools/clang-format/ClangFormat.cpp
+++ tools/clang-format/ClangFormat.cpp
@@ -243,7 +243,7 @@
 static bool format(StringRef FileName) {
   if (!OutputXML && Inplace && FileName == "-") {
 errs() << "error: cannot use -i when reading from stdin.\n";
-return false;
+return true;
   }
   // On Windows, overwriting a file with an open file mapping doesn't work,
   // so read the whole file into memory when formatting in-place.
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits