I made the changes needed to use UTF-32 instead. It turned out that the PCRE version 1 API I was using does not properly support UTF-32 patterns (only match data). Thus, I changed the code to use version 2 instead.
I have attached the two files that I changed. It works, as can be seen in the below example, but it's nowhere near complete. * '(..)..(..)⍱$' ⎕RE "footesting⌽⍱"* ┏→━━━━━━━━┓ ┃"st" "g⌽"┃ ┗∊━━━━━━━━┛ Now, there are two changes I would like to see: - If the right-hand argument is an array of strings, the pattern should be applied to all strings, collecting the results into a 2D array. This will be quite efficient, since the pattern only needs to be compiled once. - I'd like an axis-argument with options. One of those options should be a flag that causes a mismatch to yield an error instead of ⍬. This would be useful when the regex check is used to extract data out of data which is expected to follow a given pattern (think one-liners in interactive mode). The reason I haven't implemented these myself is because I find the current code to be absolutely awful, especially with all the duplicated code to deallocate PCRE structures. In Lisp I'd use an UNWIND-PROTECT (or try/finally in Java), but in C++ I think I have to declare a new class with a destructor to handle this, correct? Is there anyone who would like to clean this up? Regards, Elias On 21 September 2017 at 19:39, Juergen Sauermann < juergen.sauerm...@t-online.de> wrote: > Hi Elias, > > the UTF8_constructors look OK, but it can be tricky to properly interpret > indices (the elements of sub in your code) of > UTF8-encoded strings (i.e whether they mean code points or byte offsets). > > My feeling is that you should avoid UTF8_strings completely and go for the > UTF32 option of the library (assuming that > UTF32 are codepoints encoded as 32 bit integers). APL character strings > are almost UTF32 strings (except for gaps between > the codepoints) and they avoid all the bits shifting needed for UTF8 > strings. > > Best Regards, > /// Jürgen > > > On 09/21/2017 12:09 PM, Elias Mårtenson wrote: > > I've implemented the bare minimal needed to get regexes working through > a ⎕RE function. I've attached the diff. > > I really need Jürgen to take a look at this, since my code that constructs > the return value cannot possibly be correct. There must be a better way to > handle this which does not involve conversion back and forth between > std::string. > > Also, I have the result in an UTF-8-encoded C string, and I try to create > an UTF8_string from it like this: > > Value_P field_value(UTF8_string(field.c_str()), LOC); > > However, when I test this in APL I get the following result: > > '(..)..(..)$' ⎕RE 'sdklfjfj⍉' > ┏→━━━━━━━━━━┓ > ┃"lf" "jâ\215\211"┃ > ┗∊━━━━━━━━━━┛ > > It seems the UTF-8 conversion is not done correctly by the UTF8_string > constructor. What did I do wrong? > > Regards, > Elias > > On 21 September 2017 at 11:38, Xiao-Yong Jin <jinxiaoy...@gmail.com> > wrote: > >> >> > On Sep 20, 2017, at 9:19 PM, Peter Teeson <peter.tee...@icloud.com> >> wrote: >> > >> > (These days performance can hardly be a compelling argument >> > with multiple many-core CPU chips.) >> >> This kind of argument for APL is exactly why Fortran is still alive and >> well. >> >> > >
#include "Quad_RE.hh" #include "Workspace.hh" #include "PointerCell.hh" #define PCRE2_CODE_UNIT_WIDTH 32 #include <pcre2.h> Quad_RE Quad_RE::_fun; Quad_RE *Quad_RE::fun = &Quad_RE::_fun; static const PCRE2_UCHAR32 *ucs_to_codepoints(const UCS_string &string) { int size = string.size(); PCRE2_UCHAR32 *buf = new PCRE2_UCHAR32[size]; PCRE2_UCHAR32 *p = buf; UCS_string::iterator i = string.begin(); while(i.more()) { *p++ = i.next(); } return buf; } static UCS_string make_ucs_string(PCRE2_UCHAR32 *buf) { UCS_string result; PCRE2_UCHAR32 *p = buf; while(*p != 0) { result.append(static_cast<Unicode>(*p++)); } return result; } Quad_RE::Quad_RE() : QuadFunction(TOK_Quad_RE) { } Token Quad_RE::eval_AB(Value_P A, Value_P B) { if(!A->is_char_string()) { MORE_ERROR() << "Regexp argument must be a string value"; VALUE_ERROR; } UCS_string pattern = A->get_UCS_ravel(); const PCRE2_UCHAR32 *pattern_ucs = ucs_to_codepoints(pattern); int error_code; PCRE2_SIZE error_offset; pcre2_code *code = pcre2_compile_32(pattern_ucs, pattern.size(), PCRE2_NO_UTF_CHECK, &error_code, &error_offset, NULL); delete[] pattern_ucs; if(code == NULL) { PCRE2_UCHAR32 buf[256]; pcre2_get_error_message_32(error_code, buf, sizeof(buf)); UCS_string error_message = make_ucs_string(buf); MORE_ERROR() << "Error compiling regex at offset: " << error_offset << ": " << error_message; VALUE_ERROR; } const Shape &shape = B->get_shape(); if(shape.get_rank() == 0) { return Token(TOK_APL_VALUE1, Idx0(LOC)); } else if(B->is_char_string()) { UCS_string matched = B->get_UCS_ravel(); const PCRE2_UCHAR32 *matched_ucs = ucs_to_codepoints(matched); pcre2_match_data *match_data = pcre2_match_data_create_from_pattern_32(code, NULL); int match_result = pcre2_match_32(code, matched_ucs, matched.size(), 0, 0, match_data, NULL); if(match_result < 0) { delete[] matched_ucs; pcre2_match_data_free(match_data); pcre2_code_free(code); return Token(TOK_APL_VALUE1, Idx0(LOC)); } PCRE2_SIZE *ovector = pcre2_get_ovector_pointer_32(match_data); if(match_result == 0) { delete[] matched_ucs; pcre2_match_data_free(match_data); pcre2_code_free(code); MORE_ERROR() << "Match buffer too small"; FIXME; } if(match_result == 1) { UCS_string result(reinterpret_cast<const Unicode *>(matched_ucs + ovector[0]), ovector[1] - ovector[0]); delete[] matched_ucs; pcre2_match_data_free(match_data); pcre2_code_free(code); Value_P result_value(result, LOC); result_value->check_value(LOC); return Token(TOK_APL_VALUE1, result_value); } else { Shape shape(match_result - 1); Value_P result_value(shape, LOC); for(int i = 1 ; i < match_result ; i++) { PCRE2_SIZE start = ovector[i * 2]; PCRE2_SIZE end = ovector[i * 2 + 1]; Value_P field_value(UCS_string(reinterpret_cast<const Unicode *>(matched_ucs + start), end - start), LOC); field_value->check_value(LOC); new (result_value->next_ravel()) PointerCell(field_value, result_value.getref()); } delete[] matched_ucs; pcre2_match_data_free(match_data); pcre2_code_free(code); result_value->check_value(LOC); return Token(TOK_APL_VALUE1, result_value); } } else { pcre2_code_free(code); VALENCE_ERROR; } #if 0 const Shape &shape = B->get_shape(); if(shape.get_rank() == 0) { return Token(TOK_APL_VALUE1, Str0(LOC)); } else if(B->is_char_string()) { UCS_string matched = B->get_UCS_ravel(); cout << "will match: '" << matched << "' against '" << pattern << "'" << endl; int *matched_ucs = ucs_to_codepoints(matched); int match_result = pcre_exec(code, extra, reinterpret_cast<char *>(matched_ucs), matched.size(), 0, 0, sub, sizeof(sub) / sizeof(sub[0])); cout << "n = " << match_result << endl; if(match_result < 0) { delete[] matched_ucs; return Token(TOK_APL_VALUE1, Idx0(LOC)); } if(match_result == 1) { // No subexpressions, return the entire matched string UCS_string result(reinterpret_cast<Unicode *>(matched_ucs + sub[0]), sub[1] - sub[0]); delete[] matched_ucs; Value_P result_value(result, LOC); result_value->check_value(LOC); return Token(TOK_APL_VALUE1, result_value); } else { Shape shape(match_result - 1); Value_P result_value(shape, LOC); for(int i = 1 ; i < match_result ; i++) { size_t start = sub[i * 2]; size_t end = sub[i * 2 + 1]; Value_P field_value(UCS_string(reinterpret_cast<Unicode *>(matched_ucs + start), end - start), LOC); field_value->check_value(LOC); new (result_value->next_ravel()) PointerCell(field_value, result_value.getref()); } delete[] matched_ucs; result_value->check_value(LOC); return Token(TOK_APL_VALUE1, result_value); } } else { VALUE_ERROR; } #endif } Token Quad_RE::eval_AXB(const Value_P A, const Value_P X, const Value_P B) { return Token( TOK_APL_VALUE1, Str0( LOC ) ); } Token Quad_RE::eval_B(Value_P B) { return Token( TOK_APL_VALUE1, Str0( LOC ) ); } Token Quad_RE::eval_XB(Value_P X, Value_P B) { return Token( TOK_APL_VALUE1, Str0( LOC ) ); }
ax_path_lib_pcre.m4
Description: application/m4