Re: WIP Incremental JSON Parser

[Jacob Champion]

On Mon, Feb 26, 2024 at 7:08 AM Jacob Champion
<jacob.champ...@enterprisedb.com> wrote:
> As a brute force example of the latter, with the attached diff I get
> test failures at chunk sizes 1, 2, 3, 4, 6, and 12.

But this time with the diff.

--Jacob

diff --git 
a/src/test/modules/test_json_parser/t/001_test_json_parser_incremental.pl 
b/src/test/modules/test_json_parser/t/001_test_json_parser_incremental.pl
index 8eeb7f5b91..08c280037f 100644
--- a/src/test/modules/test_json_parser/t/001_test_json_parser_incremental.pl
+++ b/src/test/modules/test_json_parser/t/001_test_json_parser_incremental.pl
@@ -10,11 +10,13 @@ my $test_file = "$FindBin::RealBin/../tiny.json";
 
 my $exe = "test_json_parser_incremental";
 
-my ($stdout, $stderr) = run_command( [$exe, $test_file] );
-
-ok($stdout =~ /SUCCESS/, "test succeeds");
-ok(!$stderr, "no error output");
+for (my $size = 64; $size > 0; $size--)
+{
+       my ($stdout, $stderr) = run_command( [$exe, "-c", $size, $test_file] );
 
+       ok($stdout =~ /SUCCESS/, "chunk size $size: test succeeds");
+       ok(!$stderr, "chunk size $size: no error output");
+}
 
 done_testing();
 
diff --git a/src/test/modules/test_json_parser/test_json_parser_incremental.c 
b/src/test/modules/test_json_parser/test_json_parser_incremental.c
index dee5c6f7d1..a94cc8adb8 100644
--- a/src/test/modules/test_json_parser/test_json_parser_incremental.c
+++ b/src/test/modules/test_json_parser/test_json_parser_incremental.c
@@ -12,6 +12,7 @@
  * the parser in very small chunks. In practice you would normally use
  * much larger chunks, but doing this makes it more likely that the
  * full range of incement handling, especially in the lexer, is exercised.
+ * If the "-c SIZE" option is provided, that chunk size is used instead.
  *
  * The argument specifies the file containing the JSON input.
  *
@@ -34,12 +35,19 @@ main(int argc, char **argv)
        JsonLexContext lex;
        StringInfoData json;
        int                     n_read;
+       size_t          chunk_size = 60;
+
+       if (strcmp(argv[1], "-c") == 0)
+       {
+               sscanf(argv[2], "%zu", &chunk_size);
+               argv += 2;
+       }
 
        makeJsonLexContextIncremental(&lex, PG_UTF8, false);
        initStringInfo(&json);
 
        json_file = fopen(argv[1], "r");
-       while ((n_read = fread(buff, 1, 60, json_file)) > 0)
+       while ((n_read = fread(buff, 1, chunk_size, json_file)) > 0)
        {
                appendBinaryStringInfo(&json, buff, n_read);
                appendStringInfoString(&json, "1+23 trailing junk");