[
https://issues.apache.org/jira/browse/TRAFODION-1473?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14908211#comment-14908211
]
ASF GitHub Bot commented on TRAFODION-1473:
-------------------------------------------
Github user sandhyasun commented on a diff in the pull request:
https://github.com/apache/incubator-trafodion/pull/89#discussion_r40444499
--- Diff: core/sql/exp/ExpLOBaccess.cpp ---
@@ -1599,203 +1627,290 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
return LOB_DESC_READ_ERROR;
}
-
-
-
- if (!fdData_ || (openFlags_ != O_RDONLY))
- {
- hdfsCloseFile(fs_, fdData_);
- fdData_=NULL;
- openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
- }
- }
-
- if (!multipleChunks)
- {
- if ((bytesRead = hdfsPread(fs_, fdData_, desc.getOffset() + offset,
- memAddr, sizeToRead)) == -1) {
-
- return LOB_DATA_READ_ERROR;
- }
-
-
- operLen = bytesRead;
- return LOB_OPER_OK;
- }
- else
- {
- //handle reading the multiple chunks like a cursor
- err = readCursor(memAddr,size, getRequest()->getHandleIn(),
- getRequest()->getHandleInLen(), operLen);
-
-
- if (err==LOB_OPER_OK)
- closeCursor(getRequest()->getHandleIn(),
- getRequest()->getHandleInLen());
- else
- return err;
- }
- return LOB_OPER_OK;
-}
-/*
-Ex_Lob_Error ExLob::readDataFromFile(char *memAddr, Int64 len, Int64
&operLen)
-{
- int bytesRead;
-
-#ifdef SQ_USE_HDFS
- if (!fdData_ || (openFlags_ != O_RDONLY))
- {
+ if (fdData_)// we may have a stale handle. close and open to refresh
+ {
hdfsCloseFile(fs_, fdData_);
fdData_=NULL;
openFlags_ = O_RDONLY;
fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
+ openFlags_ = -1;
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+ }
+ else
+ {
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+ if (!fdData_) {
+ openFlags_ = -1;
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+ }
+
+ if (!multipleChunks)
+ {
+ if ((bytesRead = hdfsPread(fs_, fdData_, offset,
+ memAddr, size)) == -1) {
+
+ return LOB_DATA_READ_ERROR;
}
- }
- bytesRead = hdfsPread(fs_, fdData_, 0, memAddr, len);
-#else
- bytesRead = pread(fdData_, memAddr, len, 0);
-#endif
- if (bytesRead == -1) {
- operLen = 0;
- return LOB_DATA_READ_ERROR;
- } else {
+
operLen = bytesRead;
- }
-
- return LOB_OPER_OK;
-}
-
-*/
-Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset,
Int64 &operLen)
-{
- Ex_Lob_Error err;
-
- ExLobDesc desc;
- err = getDesc(desc);
- if (err != LOB_OPER_OK)
- return err;
-
- if (offset > desc.getSize())
- {
- operLen = 0;
return LOB_OPER_OK;
}
-
- char *data = (char *)memalign(512, desc.getSize());
- if (data == NULL) {
- return LOB_SOURCE_DATA_ALLOC_ERROR;
+ else
+ {
+ //handle reading the multiple chunks like a cursor
+ err = readCursor(memAddr,size, getRequest()->getHandleIn(),
+ getRequest()->getHandleInLen(), operLen);
+
+
+ if (err==LOB_OPER_OK)
+ closeCursor(getRequest()->getHandleIn(),
+ getRequest()->getHandleInLen());
+ else
+ return err;
}
+ return LOB_OPER_OK;
+}
-#ifdef SQ_USE_HDFS
- // tbd
-#else
- if (pread(fdData_, data, desc.getSize(), desc.getOffset()) == -1) {
- free(data);
- return LOB_DATA_READ_ERROR;
- }
-#endif
+
- int openFlags = O_CREAT | O_RDWR ; // O_DIRECT needs mem alignment
- int filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH |
S_IWOTH;
+Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset,
Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags)
+{
+ Ex_Lob_Error err;
+ Int64 operLen = 0;
+
+ Int64 srcLen = size;
+ Int64 srcOffset = offset;
+ Int64 tgtOffset = 0;
+ char *lobData = 0;
+ Int64 chunkSize = 0;
+ if (srcLen <=0)
+ return LOB_SOURCE_DATA_ALLOC_ERROR;
+ // open the targte file for writing
+ int filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH |
S_IWOTH;
+ int openFlags = O_RDWR ; // O_DIRECT needs mem alignment
+ if ((LobTgtFileFlags)fileflags == Lob_Append_Or_Error )
+ openFlags |= O_APPEND;
+ else
+ openFlags |= O_TRUNC;
int fdDestFile = open(fileName, openFlags, filePerms);
- if (fdDestFile == -1) {
- return LOB_DEST_FILE_OPEN_ERROR;
- }
-
- if (pwrite(fdDestFile, data, desc.getSize(), 0) == -1) {
- free(data);
- close(fdDestFile);
- return LOB_DATA_WRITE_ERROR;
- }
+
+ if (fdDestFile >0 )
--- End diff --
You're right. Will change the check.
> Adding external buffer and file input and extract support for LOB datatypes
> -----------------------------------------------------------------------------
>
> Key: TRAFODION-1473
> URL: https://issues.apache.org/jira/browse/TRAFODION-1473
> Project: Apache Trafodion
> Issue Type: Task
> Components: sql-cmp, sql-exe
> Environment: Trafodion
> Reporter: Sandhya Sundaresan
> Assignee: Sandhya Sundaresan
> Labels: features
> Fix For: 2.0-incubating
>
>
> LOB datatype is disabled in Trafodion until the following support is fully
> in.
> Here are the list of tasks and items to support :
> 1. Support input of linux files (on platform) into LOB columns.
> 2. Support extract of LOB data into linux files on platform.
> 3. Support input of hdfs files into LOB columns.
> 4. Support extract of LOB data into hdfs files.
> 5. Support chunking to input a very large external file into LOB columns.
> 6. Use CQD concept to limit LOB max size and limit chunk size for extract
> /input.
> TBD
> 7. Support for input/extract of external files off platform into LOB columns.
> 8. Support in connectivity and driver for real LOB data type.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
