-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 On 4/26/13 2:20 PM, Bro Tracker wrote: > #985: 'tail -f' functionality for file reading in input framework > ------------------------+----------------------------- Reporter: > scampbell | Type: Feature Request Status: new | > Priority: Low Milestone: Bro2.2 | Component: Bro Version: > git/master | Keywords: > ------------------------+----------------------------- With the > current input framework, file data -> event translation requires > that the entire data file be read at bro start time. This can be > prohibitive when the file sizes become large ( > 1GB ). > > It would be great to see a file open option that would start > reading at the end of the file. > I tried to update the ticket, but there seems to be issues with the bug tracker.
The patch to support this functionality is attached - it is only a few lines. An example of using this looks like: > Input::add_event([$source=data_file, $reader=Input::READER_RAW, > $mode=Input::TSTREAM, $name="issh", $fields=lineVals, > $ev=sshLine]); thanks! scott -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.9 (Darwin) Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/ iD8DBQFRfCliK2Plq8B7ZBwRAsmKAJ963D3BOpuhFKxCFAZFLEUuJAxBKQCeOW+5 WOneupNCMl2ewnwbsJ00Oqs= =NkuM -----END PGP SIGNATURE-----
*** /tmp/bro/src/input/Manager.cc 2013-04-27 12:13:56.009913562 -0700
--- bro/src/input/Manager.cc 2013-04-27 10:31:31.723948100 -0700
***************
*** 315,320 ****
--- 315,324 ----
rinfo->mode = MODE_STREAM;
break;
+ case 3:
+ rinfo->mode = MODE_TSTREAM;
+ break;
+
default:
reporter->InternalError("unknown reader mode");
}
*** /tmp/bro/src/input/ReaderBackend.h 2013-04-27 12:13:56.009913562 -0700
--- bro/src/input/ReaderBackend.h 2013-04-27 12:19:07.533907736 -0700
***************
*** 36,41 ****
--- 36,48 ----
*/
MODE_STREAM,
+ /**
+ * Streaming read tail mode. The reader should monitor the data source
+ * for new appended data. When new data is appended is has to be sent
+ * using the Put api functions. Reading begins at the *end) of the
file.
+ */
+ MODE_TSTREAM,
+
/** Internal dummy mode for initialization. */
MODE_NONE
};
*** /tmp/bro/src/input/readers/Raw.cc 2013-04-27 12:13:56.011918627 -0700
--- bro/src/input/readers/Raw.cc 2013-04-27 12:21:48.121927531 -0700
***************
*** 63,72 ****
}
}
// This is defined in input/fdstream.h
in = new boost::fdistream(fileno(file));
! if ( execute && Info().mode == MODE_STREAM )
fcntl(fileno(file), F_SETFL, O_NONBLOCK);
return true;
--- 63,76 ----
}
}
+ // If looking at file tail, reset fd to end
+ if ( Info().mode == MODE_TSTREAM )
+ fseek(file, 0, SEEK_END);
+
// This is defined in input/fdstream.h
in = new boost::fdistream(fileno(file));
! if ( execute && ((Info().mode == MODE_STREAM) || (Info().mode ==
MODE_TSTREAM)) )
fcntl(fileno(file), F_SETFL, O_NONBLOCK);
return true;
***************
*** 211,217 ****
case MODE_MANUAL:
case MODE_STREAM:
! if ( Info().mode == MODE_STREAM && file != NULL && in
!= NULL )
{
//fpurge(file);
in->clear(); // remove end of file evil bits
--- 215,222 ----
case MODE_MANUAL:
case MODE_STREAM:
! case MODE_TSTREAM:
! if ( ((Info().mode == MODE_STREAM) || (Info().mode ==
MODE_TSTREAM)) && file != NULL && in != NULL )
{
//fpurge(file);
in->clear(); // remove end of file evil bits
***************
*** 261,266 ****
--- 266,272 ----
case MODE_REREAD:
case MODE_STREAM:
+ case MODE_TSTREAM:
#ifdef DEBUG
Debug(DBG_INPUT, "Starting Heartbeat update");
#endif
*** /tmp/bro/src/types.bif 2013-04-27 12:13:56.022986957 -0700
--- bro/src/types.bif 2013-04-27 12:26:39.779955302 -0700
***************
*** 208,213 ****
--- 208,214 ----
MANUAL = 0,
REREAD = 1,
STREAM = 2,
+ TSTREAM = 3,
%}
module GLOBAL;
PATCH.sig
Description: Binary data
_______________________________________________ bro-dev mailing list [email protected] http://mailman.icsi.berkeley.edu/mailman/listinfo/bro-dev
