Index: pgadmin/include/frm/frmStatus.h =================================================================== --- pgadmin/include/frm/frmStatus.h (revision 7895) +++ pgadmin/include/frm/frmStatus.h (working copy) @@ -101,6 +101,9 @@ wxDateTime logfileTimestamp, latestTimestamp; wxString logDirectory, logfileName; + + wxString savedPartialLine; + bool showCurrent, isCurrent; long backend_pid; @@ -191,4 +194,164 @@ bool CheckEnable(pgObject *obj); }; +// PostgreSQL and GPDB now support CSV format logs. +// So, we need a way to parse the CSV files into lines, and lines into tokens. + +#include "utils/sysLogger.h" + +class CSVTokenizer : public wxObject +{ +public: + CSVTokenizer(const wxString& str) + { + m_string = str; + m_pos = 0; + } + + bool HasMoreTokens() const + { + if ( m_string.length() > 0) + { + if ( m_pos >= m_string.length()) + return false; + + if ( m_string.find_first_not_of(wxT(','), m_pos) != wxString::npos ) + // there are non delimiter characters left, so we do have more tokens + return true; + + if (m_string[m_pos] == wxT('\n')) + return false; + } + return m_pos == 0 && !m_string.empty(); + } + + // get the next token, will return empty string if !HasMoreTokens() + wxString GetNextToken() + { + wxString token; + + if ( !HasMoreTokens() ) + return token; + + // skip leading blanks if not quoted. + while (m_pos < m_string.length() && m_string[m_pos] == wxT(' ')) + m_pos ++; + + // Are we a quoted field? Must handle this special. + bool quoted_string = (m_string[m_pos] == wxT('\"')); + bool inquote = false; + + size_t pos = m_pos; + + // find the end of this token. + for (; pos < m_string.length(); pos++) + { + if (quoted_string && m_string[pos] == wxT('\"')) + inquote = !inquote; + + if (!inquote) + { + // Check to see if we have found the end of this token. + // Tokens normally end with a ',' delimiter. + if (m_string[pos] == wxT(',')) + break; + + // Last token is delimited by '\n' or by end of string. + if (m_string[pos] == wxT('\n') && pos == m_string.length()-1) + break; + } + } + + if (quoted_string && !inquote) + { + token.assign(m_string, m_pos + 1, pos - m_pos - 2); // Remove leading and trailing quotes + + // Remove double doublequote chars, replace with single doublequote chars + token.Replace(wxT("\"\""),wxT("\""),true); + } + else + token.assign(m_string, m_pos, pos - m_pos); + + if (quoted_string && inquote) + wxLogNotice(wxT("unterminated double quoted string: %s\n"),token); + + m_pos = pos + 1; // Skip token and delimiter + + if (m_pos > m_string.length()) // Perhaps no delimiter if at end of string if orig string didn't have '\n'. + m_pos = m_string.length(); + + return token; + } + +protected: + + wxString m_string; // the string we tokenize + size_t m_pos; // the current position in m_string +}; + +class CSVLineTokenizer : public wxObject +{ +public: + CSVLineTokenizer(const wxString& str) + { + m_string = str; + m_pos = 0; + } + + bool HasMoreLines() const + { + if ( m_string.find_first_not_of(wxT('\n'), m_pos) != wxString::npos ) + // there are non line-end characters left, so we do have more lines + return true; + return false; + } + + // get the next line, will return empty string if !HasMoreLines() + wxString GetNextLine(bool & partial) + { + wxString token; + partial = true; + + if ( !HasMoreLines() ) + return token; + + // find the end of this line. CSV lines end in "\n", but + // CSV lines may have "\n" chars inside double-quoted strings, so we need to find that out. + + bool inquote = false; + for (size_t pos = m_pos; pos < m_string.length(); pos++) + { + if (m_string[pos] == wxT('\"')) + inquote = !inquote; + + if (m_string[pos] == wxT('\n') && !inquote) + { + // Good, we found a complete log line terminated + // by "\n", and the "\n" wasn't in a quoted string. + + size_t len = pos - m_pos + 1; // return the line, including the trailing "\n" + token.assign(m_string, m_pos, len); + m_pos = pos + 1; // point to next line. + partial = false; + return token; + } + } + + // no more delimiters, so the line is everything till the end of + // string, but we don't have all of the CSV the line... Some must still be coming. + + token.assign(m_string, m_pos, wxString::npos); + partial = true; + + m_pos = m_string.length(); + + return token; + } + +protected: + + wxString m_string; // the string we tokenize + size_t m_pos; // the current position in m_string +}; + #endif Index: pgadmin/frm/frmMainConfig.cpp =================================================================== --- pgadmin/frm/frmMainConfig.cpp (revision 7895) +++ pgadmin/frm/frmMainConfig.cpp (working copy) @@ -298,7 +298,7 @@ { pgSettingItem *item = options[cfgList->GetText(i)]; - if (item && item->newLine && !item->orgLine) + if (item && item->newLine && item->newLine->item && !item->orgLine) str.Append(item->newLine->GetNewText() + wxT("\n")); } Index: pgadmin/frm/frmStatus.cpp =================================================================== --- pgadmin/frm/frmStatus.cpp (revision 7895) +++ pgadmin/frm/frmStatus.cpp (working copy) @@ -282,6 +282,13 @@ { manager.GetPane(wxT("Transactions")).Show(false); } + else if (connection->GetIsGreenplum()) + { + // GPDB doesn't have external global transactions. + // Perhaps we should use this display to show our + // global xid to local xid mappings? + manager.GetPane(wxT("Transactions")).Show(false); + } // Tell the manager to "commit" all the changes just made manager.Update(); @@ -601,7 +608,6 @@ { logFormatKnown = true; logHasTimestamp = true; - logList->AddColumn(_("Timestamp"), 100); } else if (connection->GetIsGreenplum()) { @@ -609,14 +615,31 @@ // Always CSV formatted for 3.3 logFormatKnown = true; logHasTimestamp = true; - logList->AddColumn(_("Timestamp"), 120); // Room for millisecs } - if (logFormatKnown) + + if (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8,2,13)) + { + // GPDB CSV format log file + logList->AddColumn(_("Timestamp"), 120); // Room for millisecs logList->AddColumn(_("Level"), 35); + logList->AddColumn(_("Log entry"), 400); + logList->AddColumn(_("Connection"), 45); + logList->AddColumn(_("Cmd number"), 48); + logList->AddColumn(_("Dbname"), 48); + logList->AddColumn(_("Segment"), 45); + } + else // Non-GPDB or non-CSV format log + { + if (logHasTimestamp) + logList->AddColumn(_("Timestamp"), 100); - logList->AddColumn(_("Log entry"), 800); + if (logFormatKnown) + logList->AddColumn(_("Level"), 35); + logList->AddColumn(_("Log entry"), 800); + } + if (!connection->HasFeature(FEATURE_ROTATELOG)) btnRotateLog->Disable(); @@ -938,7 +961,7 @@ wxCriticalSectionLocker lock(gs_critsect); - connection->ExecuteVoid(wxT("SET log_statement='none';")); + connection->ExecuteVoid(wxT("SET log_statement='none';SET log_duration='off';")); long row=0; pgSet *dataSet1=connection->ExecuteSet(wxT("SELECT *,(SELECT min(pid) FROM pg_locks l1 WHERE GRANTED AND relation IN (SELECT relation FROM pg_locks l2 WHERE l2.pid=procpid AND NOT granted)) AS blockedby FROM pg_stat_activity ORDER BY procpid")); @@ -963,7 +986,7 @@ break; } - if (!itempid || itempid > pid) + if (!itempid || itempid > pid || row >= statusList->GetItemCount()) { statusList->InsertItem(row, NumToStr(pid), 0); } @@ -1111,7 +1134,7 @@ break; } - if (!itempid || itempid > pid) + if (!itempid || itempid > pid || lockList->GetItemCount() == 0) { lockList->InsertItem(row, NumToStr(pid), 0); } @@ -1286,6 +1309,7 @@ logDirectory = connection->ExecuteScalar(wxT("SHOW log_directory")); if (fillLogfileCombo()) { + savedPartialLine.Clear(); cbLogfiles->SetSelection(0); wxCommandEvent ev; OnLoadLogfile(ev); @@ -1430,9 +1454,22 @@ skipFirst=false; } + // If GPDB 3.3 and later, log is in CSV format. Let's get a whole log line before calling addLogLine, + // so we can do things smarter. + bool gpdb_csv_format = (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13)); + + if (gpdb_csv_format && savedPartialLine.length() > 0) + { + if (read == 0) // Starting at beginning of log file? + savedPartialLine.clear(); + else + line = savedPartialLine; + } + while (len > read) { + statusBar->SetStatusText(_("Reading log from server.")); pgSet *set=connection->ExecuteSet(wxT("SELECT pg_file_read(") + connection->qtDbString(filename) + wxT(", ") + NumToStr(read) + wxT(", 50000)")); if (!set) @@ -1447,6 +1484,7 @@ delete set; break; } + read += strlen(raw); wxString str; @@ -1464,48 +1502,77 @@ return; } - bool hasCr = (str.Right(1) == wxT("\n")); + if (gpdb_csv_format) + { + // This would work for any DB using CSV format logs - wxStringTokenizer tk(str, wxT("\n")); - - bool gpdbformat = (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13)); - - logList->Freeze(); - if (gpdbformat) - { - // This would actually work for any DB assuming the log line prefix starts with %t or %m - wxString nextStr; - while (tk.HasMoreTokens()) + if (logHasTimestamp) { - str = nextStr; - nextStr = tk.GetNextToken(); - if (str.Length() == 0) - continue; + // This would actually work for any DB using CSV format logs assuming the log line prefix starts with %t or %m - // The first field must be a Timestamp, so must start with "2009" or "201" (at least for the next 10 years). - if (skipFirst && (str.Left(4) != wxT("2009") && str.Left(3) != wxT("201"))) + // and the only reason we need that is to make sure we are in sync. Bad things happen if we start in the middle of a + // double-quoted string, as we would never find a correct line terminator! + + // In GPDB, the first field must be a Timestamp, so must start with "2009" or "201" or "202" (at least for the next 20 years). + if (str.length() > 4 && str.Left(4) != wxT("2009") && str.Left(3) != wxT("201") && str.Left(3) != wxT("202")) { + wxLogError(wxT("Log line does not start with timestamp?: %s \n"), str.Mid(0,100).c_str()); // Something isn't right, as we are not at the beginning of a log record. - skipFirst = false; - continue; + // We should never get here. + str = str.Mid(str.Find(wxT("\n20"))+1); // Try to re-sync. } + } - // The first field must be a Timestamp, so must start with "2009" or "201" (at least for the next 10 years). - // If nextStr doesn't start with that, it must be a continuation of this line. - if (nextStr.Left(4) != wxT("2009") && nextStr.Left(3) != wxT("201")) + CSVLineTokenizer tk(str); + + logList->Freeze(); + + while (tk.HasMoreLines()) + { + line.Clear(); + + bool partial; + str = tk.GetNextLine(partial); + if (partial) { - nextStr = str + wxT("\n") + nextStr; - continue; + line = str; // Start of a log line, but not complete. Loop back, Read more data. + break; } - if (tk.HasMoreTokens() || hasCr) - addLogLine(str.Trim()); - else - line = str; + if (logHasTimestamp) + { + // The first field must be a Timestamp, so must start with "2009" or "201" or "202" (at least for the next 20 years). + + // This is just an extra check to make sure we haven't gotten out of sync with the log. + + if (str.length() > 5 && str.Left(4) != wxT("2009") && str.Left(3) != wxT("201") && str.Left(3) != wxT("202")) + { + // BUG: We are out of sync on the log + wxLogNotice(wxT("Log line does not start with timestamp: %s\n"), str.c_str());; + } + else if (str.length() < 20) + { + // BUG: We are out of sync on the log, or the log is garbled + wxLogNotice(wxT("Log line too short: %s\n"), str.c_str()); + } + } + + // Looks like we have a good complete CSV log record. + addLogLine(str.Trim()); } + + logList->Thaw(); } else { + // Non-csv format + + bool hasCr = (str.Right(1) == wxT("\n")); + + wxStringTokenizer tk(str, wxT("\n")); + + logList->Freeze(); + while (tk.HasMoreTokens()) { str = tk.GetNextToken(); @@ -1521,11 +1588,25 @@ else line = str; } + + logList->Thaw(); } - logList->Thaw(); } + + savedPartialLine.clear(); + if (!line.IsEmpty()) - addLogLine(line.Trim()); + { + // We finished reading to the end of the log file, but still have some data left + if (gpdb_csv_format) + { + savedPartialLine = line; // Save partial log line for next read of the data file. + line.Clear(); + } + else + addLogLine(line.Trim()); + } + } @@ -1535,30 +1616,37 @@ if (!logFormatKnown) logList->AppendItem(-1, str); else - { - if (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13)) + { + bool gpdb_csv_format = (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13)); + if (gpdb_csv_format) { // Greenplum 3.3 release and later: log is in CSV format + // In this case, we are always supposed to have a complete log line in csv format in str when called. if (!formatted) { // Not from log, from pgAdmin itself. logList->InsertItem(row, wxEmptyString, -1); logList->SetItem(row, 1, str.BeforeFirst(':')); - logList->SetItem(row, 2, str.AfterFirst(':').Mid(2)); + logList->SetItem(row, 2, str.AfterFirst(':')); } else { - wxStringTokenizer tk(str, wxT(",")); - wxString logTime = tk.GetNextToken(); - if (logTime.Length() < 20 || logTime[0] != wxT('2') || logTime[1] != wxT('0')) + if (str.Length() < 20 || (logHasTimestamp && (str[0] != wxT('2') || str[1] != wxT('0')))) { - // Log line does not start with a timestamp... Must be a continuation of the previous line or garbage. + // Log line too short or does not start with an expected timestamp... + // Must be a continuation of the previous line or garbage, + // or we are out of sync in our CSV handling. + // We shouldn't ever get here. logList->InsertItem(row, wxEmptyString, -1); logList->SetItem(row, 2, str); } else { + CSVTokenizer tk(str); + + // Get the fields from the CSV log. This is GPDB-specific. + wxString logTime = tk.GetNextToken(); wxString logUser = tk.GetNextToken(); wxString logDatabase = tk.GetNextToken(); wxString logPid = tk.GetNextToken(); @@ -1577,63 +1665,130 @@ wxString logSeverity = tk.GetNextToken(); wxString logState = tk.GetNextToken(); wxString logMessage = tk.GetNextToken(); - while (logMessage.Length() > 2 && logMessage[logMessage.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logMessage = logMessage + wxT(",") + tk.GetNextToken(); wxString logDetail = tk.GetNextToken(); - while (logDetail.Length() > 2 && logDetail[logDetail.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logDetail = logDetail + wxT(",") + tk.GetNextToken(); wxString logHint = tk.GetNextToken(); - while (logHint.Length() > 2 && logHint[logHint.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logHint = logHint + wxT(",") + tk.GetNextToken(); wxString logQuery = tk.GetNextToken(); - while (logQuery.Length() > 2 && logQuery[logQuery.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logQuery = logQuery + wxT(",") + tk.GetNextToken(); wxString logQuerypos = tk.GetNextToken(); wxString logContext = tk.GetNextToken(); wxString logDebug = tk.GetNextToken(); - while (logDebug.Length() > 2 && logDebug[logDebug.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logDebug = logDebug + wxT(",") + tk.GetNextToken(); wxString logCursorpos = tk.GetNextToken(); wxString logFunction = tk.GetNextToken(); wxString logFile = tk.GetNextToken(); wxString logLine = tk.GetNextToken(); wxString logStack = tk.GetNextToken(); - while (logStack.Length() > 2 && logStack[logStack.Length()-1] != wxT('\"') && tk.HasMoreTokens()) - logStack = logStack + wxT(",") + tk.GetNextToken(); - logSeverity = logSeverity.AfterFirst('\"').BeforeLast('\"'); - logMessage = logMessage.AfterFirst('\"').BeforeLast('\"'); - logStack = logStack.AfterFirst('\"').BeforeLast('\"'); + logList->InsertItem(row, logTime, -1); // Insert timestamp (with time zone) + + logList->SetItem(row, 1, logSeverity); + // Display the logMessage, breaking it into lines wxStringTokenizer lm(logMessage,wxT("\n")); - wxStringTokenizer ls(logStack,wxT("\n")); - - logList->InsertItem(row, logTime, -1); // Insert timestamp - - - logList->SetItem(row, 1, logSeverity); logList->SetItem(row, 2, lm.GetNextToken()); + + logList->SetItem(row, 3, logSession); + logList->SetItem(row, 4, logCmdcount); + logList->SetItem(row, 5, logDatabase); + if (logSegment.length() > 0 && logSegment != wxT("seg-1")) + { + logList->SetItem(row, 6, logSegment); + } + else + { + // If we are reading the masterDB log only, the logSegment won't + // have anything useful in it. Look in the logMessage, and see if the + // segment info exists in there. It will always be at the end. + if (logMessage.length() > 0 && logMessage[logMessage.length()-1] == wxT(')')) + { + int segpos = -1; + segpos = logMessage.Find(wxT("(seg")); + if (segpos <= 0) + segpos = logMessage.Find(wxT("(mir")); + if (segpos > 0) + { + logSegment = logMessage.Mid(segpos+1); + if (logSegment.Find(wxT(' ')) > 0) + logSegment = logSegment.Mid(0,logSegment.Find(wxT(' '))); + logList->SetItem(row, 6, logSegment); + } + } + } - + // The rest of the lines from the logMessage while (lm.HasMoreTokens()) { int controw=logList->GetItemCount(); logList->InsertItem(controw, wxEmptyString, -1); logList->SetItem(controw, 2, lm.GetNextToken()); } - while (ls.HasMoreTokens()) + + // Add the detail + wxStringTokenizer ld(logDetail,wxT("\n")); + while (ld.HasMoreTokens()) { int controw=logList->GetItemCount(); logList->InsertItem(controw, wxEmptyString, -1); - logList->SetItem(controw, 2, ls.GetNextToken()); + logList->SetItem(controw, 2, ld.GetNextToken()); } + + // And the hint + wxStringTokenizer lh(logHint,wxT("\n")); + while (lh.HasMoreTokens()) + { + int controw=logList->GetItemCount(); + logList->InsertItem(controw, wxEmptyString, -1); + logList->SetItem(controw, 2, lh.GetNextToken()); + } + + if (logDebug.length() > 0) + { + wxString logState3 = logState.Mid(0,3); + if (logState3 == wxT("426") || logState3 == wxT("22P") || logState3 == wxT("427") + || logState3 == wxT("42P") || logState3 == wxT("458") + || logMessage.Mid(0,9) == wxT("duration:") || logSeverity == wxT("FATAL") || logSeverity == wxT("PANIC")) + { + // If not redundant, add the statement from the debug_string + wxStringTokenizer lh(logDebug,wxT("\n")); + if (lh.HasMoreTokens()) + { + int controw=logList->GetItemCount(); + logList->InsertItem(controw, wxEmptyString, -1); + logList->SetItem(controw, 2, wxT("statement: ") + lh.GetNextToken()); + } + while (lh.HasMoreTokens()) + { + int controw=logList->GetItemCount(); + logList->InsertItem(controw, wxEmptyString, -1); + logList->SetItem(controw, 2, lh.GetNextToken()); + } + } + } + + if (logSeverity == wxT("PANIC") || + (logSeverity == wxT("FATAL") && logState != wxT("57P03") && logState != wxT("53300"))) + { + // If this is a severe error, add the stack trace. + wxStringTokenizer ls(logStack,wxT("\n")); + if (ls.HasMoreTokens()) + { + int controw=logList->GetItemCount(); + logList->InsertItem(controw, wxEmptyString, -1); + logList->SetItem(controw, 1, wxT("STACK")); + logList->SetItem(controw, 2, ls.GetNextToken()); + } + while (ls.HasMoreTokens()) + { + int controw=logList->GetItemCount(); + logList->InsertItem(controw, wxEmptyString, -1); + logList->SetItem(controw, 2, ls.GetNextToken()); + } + } } } } else if (connection->GetIsGreenplum()) { - // Greenplum 3.2 and before. + // Greenplum 3.2 and before. log_line_prefix = "%m|%u|%d|%p|%I|%X|:-" if (str.Find(':') < 0) { @@ -1644,25 +1799,24 @@ } else { - wxString logSeverity; - // Skip prefix, get message - wxString rest = str.Mid(str.Find(wxT(":-"))+1) ; - if (rest[0] == wxT('-')) - rest = rest.Mid(1); + if (formatted) + { + wxString logSeverity; + // Skip prefix, get message. In GPDB, always follows ":-". + wxString rest = str.Mid(str.Find(wxT(":-"))+1) ; + if (rest.Length() > 0 && rest[0] == wxT('-')) + rest = rest.Mid(1); - // Separate loglevel from message + // Separate loglevel from message + if (rest.Length() > 1 && rest[0] != wxT(' ') && rest.Find(':') > 0) + { + logSeverity = rest.BeforeFirst(':'); + rest = rest.AfterFirst(':').Mid(2); + } - if (rest[0] != wxT(' ') && rest.Find(':') > 0) - { - logSeverity = rest.BeforeFirst(':'); - rest = rest.AfterFirst(':').Mid(2); - } - - if (formatted) - { wxString ts = str.BeforeFirst(logFormat.c_str()[logFmtPos+2]); - if (ts.Length() < 20 || ts.Left(2) != wxT("20") || str.Find(':') < 0) + if (ts.Length() < 20 || (logHasTimestamp && (ts.Left(2) != wxT("20") || str.Find(':') < 0))) { // No Timestamp? Must be a continuation of a previous line? // Not sure if it is possible to get here. @@ -1683,11 +1837,11 @@ logList->SetItem(row, 2, rest); } } - else // Not from log, from pgAdmin itself? + else // Not from log, from pgAdmin itself. { logList->InsertItem(row, wxEmptyString, -1); logList->SetItem(row, 1, str.BeforeFirst(':')); - logList->SetItem(row, 2, str.AfterFirst(':').Mid(2)); + logList->SetItem(row, 2, str.AfterFirst(':')); } }