HDFS-9047. Retire libwebhdfs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83a47912 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83a47912 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83a47912 Branch: refs/heads/branch-2.7 Commit: 83a479124317c3913729330373a9ab82698106e2 Parents: 4c09a75 Author: Kihwal Lee <kih...@apache.org> Authored: Wed Jan 13 10:59:45 2016 -0600 Committer: Kihwal Lee <kih...@apache.org> Committed: Wed Jan 13 11:01:35 2016 -0600 ---------------------------------------------------------------------- BUILDING.txt | 1 - hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 +- .../hadoop-hdfs/src/CMakeLists.txt | 3 - .../src/contrib/libwebhdfs/CMakeLists.txt | 75 - .../libwebhdfs/resources/FindJansson.cmake | 42 - .../contrib/libwebhdfs/src/hdfs_http_client.c | 482 ------ .../contrib/libwebhdfs/src/hdfs_http_client.h | 294 ---- .../contrib/libwebhdfs/src/hdfs_http_query.c | 402 ----- .../contrib/libwebhdfs/src/hdfs_http_query.h | 240 --- .../contrib/libwebhdfs/src/hdfs_json_parser.c | 654 -------- .../contrib/libwebhdfs/src/hdfs_json_parser.h | 178 -- .../src/contrib/libwebhdfs/src/hdfs_web.c | 1538 ------------------ .../libwebhdfs/src/test_libwebhdfs_ops.c | 552 ------- .../libwebhdfs/src/test_libwebhdfs_read.c | 78 - .../libwebhdfs/src/test_libwebhdfs_threaded.c | 247 --- .../libwebhdfs/src/test_libwebhdfs_write.c | 111 -- 17 files changed, 4 insertions(+), 4900 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index b30b30e..e130250 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -11,7 +11,6 @@ Requirements: * CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac * Zlib devel (if compiling native code) * openssl devel ( if compiling native hadoop-pipes and to get the best HDFS encryption performance ) -* Jansson C XML parsing library ( if compiling libwebhdfs ) * Linux FUSE (Filesystem in Userspace) version 2.6 or above ( if compiling fuse_dfs ) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 710abf3..f6adb1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -16,6 +16,8 @@ Release 2.7.3 - UNRELEASED HDFS-9574. Reduce client failures during datanode restart (kihwal) + HDFS-9047. Retire libwebhdfs (kihwal) + HDFS-9569. Log the name of the fsimage being loaded for better supportability. (Yongjun Zhang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 2701a77..fb1f315 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -34,7 +34,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <hadoop.component>hdfs</hadoop.component> <is.hadoop.component>true</is.hadoop.component> <require.fuse>false</require.fuse> - <require.libwebhdfs>false</require.libwebhdfs> </properties> <dependencies> @@ -449,7 +448,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <mkdir dir="${project.build.directory}/native"/> <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true"> - <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse} -G '${generator}'"/> + <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_FUSE=${require.fuse} -G '${generator}'"/> </exec> <exec executable="msbuild" dir="${project.build.directory}/native" failonerror="true"> @@ -519,7 +518,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> <mkdir dir="${project.build.directory}/native"/> <exec executable="cmake" dir="${project.build.directory}/native" failonerror="true"> - <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}"/> + <arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_FUSE=${require.fuse}"/> </exec> <exec executable="make" dir="${project.build.directory}/native" failonerror="true"> <arg line="VERBOSE=1"/> http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt index 563727b..de885db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt @@ -250,7 +250,4 @@ if (NOT WIN32) endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") endif(NOT WIN32) -IF(REQUIRE_LIBWEBHDFS) - add_subdirectory(contrib/libwebhdfs) -ENDIF(REQUIRE_LIBWEBHDFS) add_subdirectory(main/native/fuse-dfs) http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt deleted file mode 100644 index 25f95b2..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt +++ /dev/null @@ -1,75 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -find_package(CURL REQUIRED) - -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} -"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/") - -find_package(Jansson REQUIRED) -include_directories(${JANSSON_INCLUDE_DIR}) - -add_dual_library(webhdfs - src/hdfs_web.c - src/hdfs_http_client.c - src/hdfs_http_query.c - src/hdfs_json_parser.c - ../../main/native/libhdfs/exception.c - ../../main/native/libhdfs/jni_helper.c -) -target_link_dual_libraries(webhdfs - ${JAVA_JVM_LIBRARY} - ${CURL_LIBRARY} - ${JANSSON_LIBRARY} - pthread -) -dual_output_directory(webhdfs target) -set(LIBWEBHDFS_VERSION "0.0.0") -set_target_properties(webhdfs PROPERTIES - SOVERSION ${LIBWEBHDFS_VERSION}) - -add_executable(test_libwebhdfs_ops - src/test_libwebhdfs_ops.c -) -target_link_libraries(test_libwebhdfs_ops - webhdfs - native_mini_dfs -) - -add_executable(test_libwebhdfs_read - src/test_libwebhdfs_read.c -) -target_link_libraries(test_libwebhdfs_read - webhdfs -) - -add_executable(test_libwebhdfs_write - src/test_libwebhdfs_write.c -) -target_link_libraries(test_libwebhdfs_write - webhdfs -) - -add_executable(test_libwebhdfs_threaded - src/test_libwebhdfs_threaded.c -) -target_link_libraries(test_libwebhdfs_threaded - webhdfs - native_mini_dfs - pthread -) http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake deleted file mode 100644 index cb8c8db..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -# - Try to find Jansson -# Once done this will define -# JANSSON_FOUND - System has Jansson -# JANSSON_INCLUDE_DIRS - The Jansson include directories -# JANSSON_LIBRARIES - The libraries needed to use Jansson -# JANSSON_DEFINITIONS - Compiler switches required for using Jansson - -find_path(JANSSON_INCLUDE_DIR jansson.h - /usr/incluce - /usr/local/include ) - -find_library(JANSSON_LIBRARY NAMES jansson - PATHS /usr/lib /usr/local/lib ) - -set(JANSSON_LIBRARIES ${JANSSON_LIBRARY} ) -set(JANSSON_INCLUDE_DIRS ${JANSSON_INCLUDE_DIR} ) - -include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set JANSSON_FOUND to TRUE -# if all listed variables are TRUE -find_package_handle_standard_args(Jansson DEFAULT_MSG - JANSSON_LIBRARY JANSSON_INCLUDE_DIR) - -mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c deleted file mode 100644 index dc58318..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.c +++ /dev/null @@ -1,482 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> -#include <string.h> -#include <curl/curl.h> - -#include "hdfs_http_client.h" -#include "exception.h" - -static pthread_mutex_t curlInitMutex = PTHREAD_MUTEX_INITIALIZER; -static volatile int curlGlobalInited = 0; - -const char *hdfs_strerror(int errnoval) -{ - return terror(errnoval); -} - -int initResponseBuffer(struct ResponseBuffer **buffer) -{ - struct ResponseBuffer *info = NULL; - int ret = 0; - info = calloc(1, sizeof(struct ResponseBuffer)); - if (!info) { - ret = ENOMEM; - } - *buffer = info; - return ret; -} - -void freeResponseBuffer(struct ResponseBuffer *buffer) -{ - if (buffer) { - if (buffer->content) { - free(buffer->content); - } - free(buffer); - buffer = NULL; - } -} - -void freeResponse(struct Response *resp) -{ - if (resp) { - freeResponseBuffer(resp->body); - freeResponseBuffer(resp->header); - free(resp); - resp = NULL; - } -} - -/** - * Callback used by libcurl for allocating local buffer and - * reading data to local buffer - */ -static size_t writefunc(void *ptr, size_t size, - size_t nmemb, struct ResponseBuffer *rbuffer) -{ - void *temp = NULL; - if (size * nmemb < 1) { - return 0; - } - if (!rbuffer) { - fprintf(stderr, - "ERROR: ResponseBuffer is NULL for the callback writefunc.\n"); - return 0; - } - - if (rbuffer->remaining < size * nmemb) { - temp = realloc(rbuffer->content, rbuffer->offset + size * nmemb + 1); - if (temp == NULL) { - fprintf(stderr, "ERROR: fail to realloc in callback writefunc.\n"); - return 0; - } - rbuffer->content = temp; - rbuffer->remaining = size * nmemb; - } - memcpy(rbuffer->content + rbuffer->offset, ptr, size * nmemb); - rbuffer->offset += size * nmemb; - (rbuffer->content)[rbuffer->offset] = '\0'; - rbuffer->remaining -= size * nmemb; - return size * nmemb; -} - -/** - * Callback used by libcurl for reading data into buffer provided by user, - * thus no need to reallocate buffer. - */ -static size_t writeFuncWithUserBuffer(void *ptr, size_t size, - size_t nmemb, struct ResponseBuffer *rbuffer) -{ - size_t toCopy = 0; - if (size * nmemb < 1) { - return 0; - } - if (!rbuffer || !rbuffer->content) { - fprintf(stderr, - "ERROR: buffer to read is NULL for the " - "callback writeFuncWithUserBuffer.\n"); - return 0; - } - - toCopy = rbuffer->remaining < (size * nmemb) ? - rbuffer->remaining : (size * nmemb); - memcpy(rbuffer->content + rbuffer->offset, ptr, toCopy); - rbuffer->offset += toCopy; - rbuffer->remaining -= toCopy; - return toCopy; -} - -/** - * Callback used by libcurl for writing data to remote peer - */ -static size_t readfunc(void *ptr, size_t size, size_t nmemb, void *stream) -{ - struct webhdfsBuffer *wbuffer = NULL; - if (size * nmemb < 1) { - return 0; - } - - wbuffer = stream; - pthread_mutex_lock(&wbuffer->writeMutex); - while (wbuffer->remaining == 0) { - /* - * The current remainning bytes to write is 0, - * check closeFlag to see whether need to finish the transfer. - * if yes, return 0; else, wait - */ - if (wbuffer->closeFlag) { // We can close the transfer now - //For debug - fprintf(stderr, "CloseFlag is set, ready to close the transfer\n"); - pthread_mutex_unlock(&wbuffer->writeMutex); - return 0; - } else { - // remaining == 0 but closeFlag is not set - // indicates that user's buffer has been transferred - pthread_cond_signal(&wbuffer->transfer_finish); - pthread_cond_wait(&wbuffer->newwrite_or_close, - &wbuffer->writeMutex); - } - } - - if (wbuffer->remaining > 0 && !wbuffer->closeFlag) { - size_t copySize = wbuffer->remaining < size * nmemb ? - wbuffer->remaining : size * nmemb; - memcpy(ptr, wbuffer->wbuffer + wbuffer->offset, copySize); - wbuffer->offset += copySize; - wbuffer->remaining -= copySize; - pthread_mutex_unlock(&wbuffer->writeMutex); - return copySize; - } else { - fprintf(stderr, "ERROR: webhdfsBuffer's remaining is %ld, " - "it should be a positive value!\n", wbuffer->remaining); - pthread_mutex_unlock(&wbuffer->writeMutex); - return 0; - } -} - -/** - * Initialize the global libcurl environment - */ -static void initCurlGlobal() -{ - if (!curlGlobalInited) { - pthread_mutex_lock(&curlInitMutex); - if (!curlGlobalInited) { - curl_global_init(CURL_GLOBAL_ALL); - curlGlobalInited = 1; - } - pthread_mutex_unlock(&curlInitMutex); - } -} - -/** - * Launch simple commands (commands without file I/O) and return response - * - * @param url Target URL - * @param method HTTP method (GET/PUT/POST) - * @param followloc Whether or not need to set CURLOPT_FOLLOWLOCATION - * @param response Response from remote service - * @return 0 for success and non-zero value to indicate error - */ -static int launchCmd(const char *url, enum HttpHeader method, - enum Redirect followloc, struct Response **response) -{ - CURL *curl = NULL; - CURLcode curlCode; - int ret = 0; - struct Response *resp = NULL; - - resp = calloc(1, sizeof(struct Response)); - if (!resp) { - return ENOMEM; - } - ret = initResponseBuffer(&(resp->body)); - if (ret) { - goto done; - } - ret = initResponseBuffer(&(resp->header)); - if (ret) { - goto done; - } - initCurlGlobal(); - curl = curl_easy_init(); - if (!curl) { - ret = ENOMEM; // curl_easy_init does not return error code, - // and most of its errors are caused by malloc() - fprintf(stderr, "ERROR in curl_easy_init.\n"); - goto done; - } - /* Set callback function for reading data from remote service */ - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, resp->body); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, writefunc); - curl_easy_setopt(curl, CURLOPT_WRITEHEADER, resp->header); - curl_easy_setopt(curl, CURLOPT_URL, url); - switch(method) { - case GET: - break; - case PUT: - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); - break; - case POST: - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); - break; - case DELETE: - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - break; - default: - ret = EINVAL; - fprintf(stderr, "ERROR: Invalid HTTP method\n"); - goto done; - } - if (followloc == YES) { - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - } - /* Now run the curl handler */ - curlCode = curl_easy_perform(curl); - if (curlCode != CURLE_OK) { - ret = EIO; - fprintf(stderr, "ERROR: preform the URL %s failed, <%d>: %s\n", - url, curlCode, curl_easy_strerror(curlCode)); - } -done: - if (curl != NULL) { - curl_easy_cleanup(curl); - } - if (ret) { - free(resp); - resp = NULL; - } - *response = resp; - return ret; -} - -/** - * Launch the read request. The request is sent to the NameNode and then - * redirected to corresponding DataNode - * - * @param url The URL for the read request - * @param resp The response containing the buffer provided by user - * @return 0 for success and non-zero value to indicate error - */ -static int launchReadInternal(const char *url, struct Response* resp) -{ - CURL *curl; - CURLcode curlCode; - int ret = 0; - - if (!resp || !resp->body || !resp->body->content) { - fprintf(stderr, - "ERROR: invalid user-provided buffer!\n"); - return EINVAL; - } - - initCurlGlobal(); - /* get a curl handle */ - curl = curl_easy_init(); - if (!curl) { - fprintf(stderr, "ERROR in curl_easy_init.\n"); - return ENOMEM; - } - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFuncWithUserBuffer); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, resp->body); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, writefunc); - curl_easy_setopt(curl, CURLOPT_WRITEHEADER, resp->header); - curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - - curlCode = curl_easy_perform(curl); - if (curlCode != CURLE_OK && curlCode != CURLE_PARTIAL_FILE) { - ret = EIO; - fprintf(stderr, "ERROR: preform the URL %s failed, <%d>: %s\n", - url, curlCode, curl_easy_strerror(curlCode)); - } - - curl_easy_cleanup(curl); - return ret; -} - -/** - * The function does the write operation by connecting to a DataNode. - * The function keeps the connection with the DataNode until - * the closeFlag is set. Whenever the current data has been sent out, - * the function blocks waiting for further input from user or close. - * - * @param url URL of the remote DataNode - * @param method PUT for create and POST for append - * @param uploadBuffer Buffer storing user's data to write - * @param response Response from remote service - * @return 0 for success and non-zero value to indicate error - */ -static int launchWrite(const char *url, enum HttpHeader method, - struct webhdfsBuffer *uploadBuffer, - struct Response **response) -{ - CURLcode curlCode; - struct Response* resp = NULL; - struct curl_slist *chunk = NULL; - CURL *curl = NULL; - int ret = 0; - - if (!uploadBuffer) { - fprintf(stderr, "ERROR: upload buffer is NULL!\n"); - return EINVAL; - } - - initCurlGlobal(); - resp = calloc(1, sizeof(struct Response)); - if (!resp) { - return ENOMEM; - } - ret = initResponseBuffer(&(resp->body)); - if (ret) { - goto done; - } - ret = initResponseBuffer(&(resp->header)); - if (ret) { - goto done; - } - - // Connect to the datanode in order to create the lease in the namenode - curl = curl_easy_init(); - if (!curl) { - fprintf(stderr, "ERROR: failed to initialize the curl handle.\n"); - return ENOMEM; - } - curl_easy_setopt(curl, CURLOPT_URL, url); - - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, resp->body); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, writefunc); - curl_easy_setopt(curl, CURLOPT_WRITEHEADER, resp->header); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, readfunc); - curl_easy_setopt(curl, CURLOPT_READDATA, uploadBuffer); - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - - chunk = curl_slist_append(chunk, "Transfer-Encoding: chunked"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); - chunk = curl_slist_append(chunk, "Expect:"); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); - - switch(method) { - case PUT: - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT"); - break; - case POST: - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); - break; - default: - ret = EINVAL; - fprintf(stderr, "ERROR: Invalid HTTP method\n"); - goto done; - } - curlCode = curl_easy_perform(curl); - if (curlCode != CURLE_OK) { - ret = EIO; - fprintf(stderr, "ERROR: preform the URL %s failed, <%d>: %s\n", - url, curlCode, curl_easy_strerror(curlCode)); - } - -done: - if (chunk != NULL) { - curl_slist_free_all(chunk); - } - if (curl != NULL) { - curl_easy_cleanup(curl); - } - if (ret) { - free(resp); - resp = NULL; - } - *response = resp; - return ret; -} - -int launchMKDIR(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchRENAME(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchGFS(const char *url, struct Response **resp) -{ - return launchCmd(url, GET, NO, resp); -} - -int launchLS(const char *url, struct Response **resp) -{ - return launchCmd(url, GET, NO, resp); -} - -int launchCHMOD(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchCHOWN(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchDELETE(const char *url, struct Response **resp) -{ - return launchCmd(url, DELETE, NO, resp); -} - -int launchOPEN(const char *url, struct Response* resp) -{ - return launchReadInternal(url, resp); -} - -int launchUTIMES(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchNnWRITE(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} - -int launchNnAPPEND(const char *url, struct Response **resp) -{ - return launchCmd(url, POST, NO, resp); -} - -int launchDnWRITE(const char *url, struct webhdfsBuffer *buffer, - struct Response **resp) -{ - return launchWrite(url, PUT, buffer, resp); -} - -int launchDnAPPEND(const char *url, struct webhdfsBuffer *buffer, - struct Response **resp) -{ - return launchWrite(url, POST, buffer, resp); -} - -int launchSETREPLICATION(const char *url, struct Response **resp) -{ - return launchCmd(url, PUT, NO, resp); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h deleted file mode 100644 index 0e70dec..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_client.h +++ /dev/null @@ -1,294 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - - -#ifndef _HDFS_HTTP_CLIENT_H_ -#define _HDFS_HTTP_CLIENT_H_ - -#include "hdfs.h" /* for tSize */ - -#include <pthread.h> /* for pthread_t */ -#include <unistd.h> /* for size_t */ - -/** enum indicating the type of hdfs stream */ -enum hdfsStreamType -{ - UNINITIALIZED = 0, - INPUT = 1, - OUTPUT = 2, -}; - -/** - * webhdfsBuffer - used for hold the data for read/write from/to http connection - */ -struct webhdfsBuffer { - const char *wbuffer; /* The user's buffer for uploading */ - size_t remaining; /* Length of content */ - size_t offset; /* offset for reading */ - /* Check whether the hdfsOpenFile has been called before */ - int openFlag; - /* Whether to close the http connection for writing */ - int closeFlag; - /* Synchronization between the curl and hdfsWrite threads */ - pthread_mutex_t writeMutex; - /* - * Transferring thread waits for this condition - * when there is no more content for transferring in the buffer - */ - pthread_cond_t newwrite_or_close; - /* Condition used to indicate finishing transferring (one buffer) */ - pthread_cond_t transfer_finish; -}; - -/** File handle for webhdfs */ -struct webhdfsFileHandle { - char *absPath; /* Absolute path of file */ - int bufferSize; /* Size of buffer */ - short replication; /* Number of replication */ - tSize blockSize; /* Block size */ - char *datanode; /* URL of the DataNode */ - /* webhdfsBuffer handle used to store the upload data */ - struct webhdfsBuffer *uploadBuffer; - /* The thread used for data transferring */ - pthread_t connThread; -}; - -/** Type of http header */ -enum HttpHeader { - GET, - PUT, - POST, - DELETE -}; - -/** Whether to redirect */ -enum Redirect { - YES, - NO -}; - -/** Buffer used for holding response */ -struct ResponseBuffer { - char *content; - size_t remaining; - size_t offset; -}; - -/** - * The response got through webhdfs - */ -struct Response { - struct ResponseBuffer *body; - struct ResponseBuffer *header; -}; - -/** - * Create and initialize a ResponseBuffer - * - * @param buffer Pointer pointing to new created ResponseBuffer handle - * @return 0 for success, non-zero value to indicate error - */ -int initResponseBuffer(struct ResponseBuffer **buffer) __attribute__ ((warn_unused_result)); - -/** - * Free the given ResponseBuffer - * - * @param buffer The ResponseBuffer to free - */ -void freeResponseBuffer(struct ResponseBuffer *buffer); - -/** - * Free the given Response - * - * @param resp The Response to free - */ -void freeResponse(struct Response *resp); - -/** - * Send the MKDIR request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for MKDIR operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchMKDIR(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the RENAME request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for RENAME operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchRENAME(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the CHMOD request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for CHMOD operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchCHMOD(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the GetFileStatus request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for GetFileStatus operation - * @param response Response handle to store response returned from the NameNode, - * containing either file status or exception information - * @return 0 for success, non-zero value to indicate error - */ -int launchGFS(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the LS (LISTSTATUS) request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for LISTSTATUS operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchLS(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the DELETE request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for DELETE operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchDELETE(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the CHOWN request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for CHOWN operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchCHOWN(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the OPEN request to NameNode using the given URL, - * asking for reading a file (within a range). - * The NameNode first redirects the request to the datanode - * that holds the corresponding first block of the file (within a range), - * and the datanode returns the content of the file through the HTTP connection. - * - * @param url The URL for OPEN operation - * @param resp The response holding user's buffer. - The file content will be written into the buffer. - * @return 0 for success, non-zero value to indicate error - */ -int launchOPEN(const char *url, - struct Response* resp) __attribute__ ((warn_unused_result)); - -/** - * Send the SETTIMES request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for SETTIMES operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchUTIMES(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the WRITE/CREATE request to NameNode using the given URL. - * The NameNode will choose the writing target datanodes - * and return the first datanode in the pipeline as response - * - * @param url The URL for WRITE/CREATE operation connecting to NameNode - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchNnWRITE(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the WRITE request along with to-write content to - * the corresponding DataNode using the given URL. - * The DataNode will write the data and return the response. - * - * @param url The URL for WRITE operation connecting to DataNode - * @param buffer The webhdfsBuffer containing data to be written to hdfs - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchDnWRITE(const char *url, struct webhdfsBuffer *buffer, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the WRITE (APPEND) request to NameNode using the given URL. - * The NameNode determines the DataNode for appending and - * sends its URL back as response. - * - * @param url The URL for APPEND operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchNnAPPEND(const char *url, struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the SETREPLICATION request to NameNode using the given URL. - * The NameNode will execute the operation and return the result as response. - * - * @param url The URL for SETREPLICATION operation - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchSETREPLICATION(const char *url, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Send the APPEND request along with the content to DataNode. - * The DataNode will do the appending and return the result as response. - * - * @param url The URL for APPEND operation connecting to DataNode - * @param buffer The webhdfsBuffer containing data to be appended - * @param response Response handle to store response returned from the NameNode - * @return 0 for success, non-zero value to indicate error - */ -int launchDnAPPEND(const char *url, struct webhdfsBuffer *buffer, - struct Response **response) __attribute__ ((warn_unused_result)); - -/** - * Call sys_errlist to get the error message string for the given error code - * - * @param errnoval The error code value - * @return The error message string mapped to the given error code - */ -const char *hdfs_strerror(int errnoval); - -#endif //_HDFS_HTTP_CLIENT_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c deleted file mode 100644 index b082c08..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c +++ /dev/null @@ -1,402 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "hdfs_http_query.h" -#include <math.h> -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include <errno.h> - -#define PERM_STR_LEN 4 // "644" + one byte for NUL -#define SHORT_STR_LEN 6 // 65535 + NUL -#define LONG_STR_LEN 21 // 2^64-1 = 18446744073709551615 + NUL - -/** - * Create query based on NameNode hostname, - * NameNode port, path, operation and other parameters - * - * @param host NameNode hostName - * @param nnPort Port of NameNode - * @param path Absolute path for the corresponding file - * @param op Operations - * @param paraNum Number of remaining parameters - * @param paraNames Names of remaining parameters - * @param paraValues Values of remaining parameters - * @param url Holding the created URL - * @return 0 on success and non-zero value to indicate error - */ -static int createQueryURL(const char *host, unsigned int nnPort, - const char *path, const char *op, int paraNum, - const char **paraNames, const char **paraValues, - char **queryUrl) -{ - size_t length = 0; - int i = 0, offset = 0, ret = 0; - char *url = NULL; - const char *protocol = "http://"; - const char *prefix = "/webhdfs/v1"; - - if (!paraNames || !paraValues) { - return EINVAL; - } - length = strlen(protocol) + strlen(host) + strlen(":") + - SHORT_STR_LEN + strlen(prefix) + strlen(path) + - strlen ("?op=") + strlen(op); - for (i = 0; i < paraNum; i++) { - if (paraNames[i] && paraValues[i]) { - length += 2 + strlen(paraNames[i]) + strlen(paraValues[i]); - } - } - url = malloc(length); // The '\0' has already been included - // when using SHORT_STR_LEN - if (!url) { - return ENOMEM; - } - - offset = snprintf(url, length, "%s%s:%d%s%s?op=%s", - protocol, host, nnPort, prefix, path, op); - if (offset >= length || offset < 0) { - ret = EIO; - goto done; - } - for (i = 0; i < paraNum; i++) { - if (!paraNames[i] || !paraValues[i] || paraNames[i][0] == '\0' || - paraValues[i][0] == '\0') { - continue; - } - offset += snprintf(url + offset, length - offset, - "&%s=%s", paraNames[i], paraValues[i]); - if (offset >= length || offset < 0) { - ret = EIO; - goto done; - } - } -done: - if (ret) { - free(url); - return ret; - } - *queryUrl = url; - return 0; -} - -int createUrlForMKDIR(const char *host, int nnPort, - const char *path, const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "MKDIRS", 1, - &userPara, &user, url); -} - -int createUrlForGetFileStatus(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "GETFILESTATUS", 1, - &userPara, &user, url); -} - -int createUrlForLS(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "LISTSTATUS", - 1, &userPara, &user, url); -} - -int createUrlForNnAPPEND(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "APPEND", - 1, &userPara, &user, url); -} - -int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, - int mode, const char *user, char **url) -{ - int strlength; - char permission[PERM_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - - paraNames[0] = "permission"; - paraNames[1] = "user.name"; - memset(permission, 0, PERM_STR_LEN); - strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); - if (strlength < 0 || strlength >= PERM_STR_LEN) { - return EIO; - } - paraValues[0] = permission; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "MKDIRS", 2, - paraNames, paraValues, url); -} - -int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, - const char *destpath, const char *user, char **url) -{ - const char *paraNames[2], *paraValues[2]; - paraNames[0] = "destination"; - paraNames[1] = "user.name"; - paraValues[0] = destpath; - paraValues[1] = user; - - return createQueryURL(host, nnPort, srcpath, - "RENAME", 2, paraNames, paraValues, url); -} - -int createUrlForCHMOD(const char *host, int nnPort, const char *path, - int mode, const char *user, char **url) -{ - int strlength; - char permission[PERM_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - - paraNames[0] = "permission"; - paraNames[1] = "user.name"; - memset(permission, 0, PERM_STR_LEN); - strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); - if (strlength < 0 || strlength >= PERM_STR_LEN) { - return EIO; - } - paraValues[0] = permission; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "SETPERMISSION", - 2, paraNames, paraValues, url); -} - -int createUrlForDELETE(const char *host, int nnPort, const char *path, - int recursive, const char *user, char **url) -{ - const char *paraNames[2], *paraValues[2]; - paraNames[0] = "recursive"; - paraNames[1] = "user.name"; - if (recursive) { - paraValues[0] = "true"; - } else { - paraValues[0] = "false"; - } - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "DELETE", - 2, paraNames, paraValues, url); -} - -int createUrlForCHOWN(const char *host, int nnPort, const char *path, - const char *owner, const char *group, - const char *user, char **url) -{ - const char *paraNames[3], *paraValues[3]; - paraNames[0] = "owner"; - paraNames[1] = "group"; - paraNames[2] = "user.name"; - paraValues[0] = owner; - paraValues[1] = group; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "SETOWNER", - 3, paraNames, paraValues, url); -} - -int createUrlForOPEN(const char *host, int nnPort, const char *path, - const char *user, size_t offset, size_t length, char **url) -{ - int strlength; - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "OPEN", - 3, paraNames, paraValues, url); -} - -int createUrlForUTIMES(const char *host, int nnPort, const char *path, - long unsigned mTime, long unsigned aTime, - const char *user, char **url) -{ - int strlength; - char modTime[LONG_STR_LEN], acsTime[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - - memset(modTime, 0, LONG_STR_LEN); - memset(acsTime, 0, LONG_STR_LEN); - strlength = snprintf(modTime, LONG_STR_LEN, "%lu", mTime); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - strlength = snprintf(acsTime, LONG_STR_LEN, "%lu", aTime); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - paraNames[0] = "modificationtime"; - paraNames[1] = "accesstime"; - paraNames[2] = "user.name"; - paraValues[0] = modTime; - paraValues[1] = acsTime; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "SETTIMES", - 3, paraNames, paraValues, url); -} - -int createUrlForNnWRITE(const char *host, int nnPort, - const char *path, const char *user, - int16_t replication, size_t blockSize, char **url) -{ - int strlength; - char repStr[SHORT_STR_LEN], blockSizeStr[LONG_STR_LEN]; - const char *paraNames[4], *paraValues[4]; - - memset(repStr, 0, SHORT_STR_LEN); - memset(blockSizeStr, 0, LONG_STR_LEN); - if (replication > 0) { - strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); - if (strlength < 0 || strlength >= SHORT_STR_LEN) { - return EIO; - } - } - if (blockSize > 0) { - strlength = snprintf(blockSizeStr, LONG_STR_LEN, "%lu", blockSize); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - paraNames[0] = "overwrite"; - paraNames[1] = "replication"; - paraNames[2] = "blocksize"; - paraNames[3] = "user.name"; - paraValues[0] = "true"; - paraValues[1] = repStr; - paraValues[2] = blockSizeStr; - paraValues[3] = user; - - return createQueryURL(host, nnPort, path, "CREATE", - 4, paraNames, paraValues, url); -} - -int createUrlForSETREPLICATION(const char *host, int nnPort, - const char *path, int16_t replication, - const char *user, char **url) -{ - char repStr[SHORT_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - int strlength; - - memset(repStr, 0, SHORT_STR_LEN); - if (replication > 0) { - strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); - if (strlength < 0 || strlength >= SHORT_STR_LEN) { - return EIO; - } - } - paraNames[0] = "replication"; - paraNames[1] = "user.name"; - paraValues[0] = repStr; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "SETREPLICATION", - 2, paraNames, paraValues, url); -} - -int createUrlForGetBlockLocations(const char *host, int nnPort, - const char *path, size_t offset, - size_t length, const char *user, char **url) -{ - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - int strlength; - - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - if (offset > 0) { - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - if (length > 0) { - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "GET_BLOCK_LOCATIONS", - 3, paraNames, paraValues, url); -} - -int createUrlForReadFromDatanode(const char *dnHost, int dnPort, - const char *path, size_t offset, - size_t length, const char *user, - const char *namenodeRpcAddr, char **url) -{ - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[4], *paraValues[4]; - int strlength; - - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - if (offset > 0) { - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - if (length > 0) { - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - paraNames[3] = "namenoderpcaddress"; - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - paraValues[3] = namenodeRpcAddr; - - return createQueryURL(dnHost, dnPort, path, "OPEN", - 4, paraNames, paraValues, url); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h deleted file mode 100644 index 432797b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef _HDFS_HTTP_QUERY_H_ -#define _HDFS_HTTP_QUERY_H_ - -#include <unistd.h> /* for size_t */ -#include <inttypes.h> /* for int16_t */ - -/** - * Create the URL for a MKDIR request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param user User name - * @param url Holding the generated URL for MKDIR request - * @return 0 on success and non-zero value on errors - */ -int createUrlForMKDIR(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a MKDIR (with mode) request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param mode Mode of MKDIR - * @param user User name - * @param url Holding the generated URL for MKDIR request - * @return 0 on success and non-zero value on errors - */ -int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, - int mode, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a RENAME request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param srcpath Source path - * @param dstpath Destination path - * @param user User name - * @param url Holding the generated URL for RENAME request - * @return 0 on success and non-zero value on errors - */ -int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, - const char *dstpath, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a CHMOD request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Target path - * @param mode New mode for the file - * @param user User name - * @param url Holding the generated URL for CHMOD request - * @return 0 on success and non-zero value on errors - */ -int createUrlForCHMOD(const char *host, int nnPort, const char *path, - int mode, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a GETFILESTATUS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param user User name - * @param url Holding the generated URL for GETFILESTATUS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForGetFileStatus(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a LISTSTATUS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the directory for listing - * @param user User name - * @param url Holding the generated URL for LISTSTATUS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForLS(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a DELETE request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file to be deletected - * @param recursive Whether or not to delete in a recursive way - * @param user User name - * @param url Holding the generated URL for DELETE request - * @return 0 on success and non-zero value on errors - */ -int createUrlForDELETE(const char *host, int nnPort, const char *path, - int recursive, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a CHOWN request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target - * @param owner New owner - * @param group New group - * @param user User name - * @param url Holding the generated URL for CHOWN request - * @return 0 on success and non-zero value on errors - */ -int createUrlForCHOWN(const char *host, int nnPort, const char *path, - const char *owner, const char *group, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a OPEN/READ request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file to read - * @param user User name - * @param offset Offset for reading (the start position for this read) - * @param length Length of the file to read - * @param url Holding the generated URL for OPEN/READ request - * @return 0 on success and non-zero value on errors - */ -int createUrlForOPEN(const char *host, int nnPort, const char *path, - const char *user, size_t offset, size_t length, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a UTIMES (update time) request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file for updating time - * @param mTime Modified time to set - * @param aTime Access time to set - * @param user User name - * @param url Holding the generated URL for UTIMES request - * @return 0 on success and non-zero value on errors - */ -int createUrlForUTIMES(const char *host, int nnPort, const char *path, - long unsigned mTime, long unsigned aTime, - const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a WRITE/CREATE request (sent to NameNode) - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param user User name - * @param replication Number of replication of the file - * @param blockSize Size of the block for the file - * @param url Holding the generated URL for WRITE request - * @return 0 on success and non-zero value on errors - */ -int createUrlForNnWRITE(const char *host, int nnPort, const char *path, - const char *user, int16_t replication, size_t blockSize, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for an APPEND request (sent to NameNode) - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file for appending - * @param user User name - * @param url Holding the generated URL for APPEND request - * @return 0 on success and non-zero value on errors - */ -int createUrlForNnAPPEND(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a SETREPLICATION request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param replication New replication number - * @param user User name - * @param url Holding the generated URL for SETREPLICATION request - * @return 0 on success and non-zero value on errors - */ -int createUrlForSETREPLICATION(const char *host, int nnPort, const char *path, - int16_t replication, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a GET_BLOCK_LOCATIONS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param offset The offset in the file - * @param length Length of the file content - * @param user User name - * @param url Holding the generated URL for GET_BLOCK_LOCATIONS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForGetBlockLocations(const char *host, int nnPort, - const char *path, size_t offset, - size_t length, const char *user, - char **url) __attribute__ ((warn_unused_result)); - - -#endif //_HDFS_HTTP_QUERY_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/83a47912/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c deleted file mode 100644 index 178fb9d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c +++ /dev/null @@ -1,654 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "exception.h" -#include "hdfs.h" /* for hdfsFileInfo */ -#include "hdfs_json_parser.h" - -#include <stdlib.h> -#include <string.h> -#include <ctype.h> -#include <jansson.h> - -static const char * const temporaryRedirectCode = "307 TEMPORARY_REDIRECT"; -static const char * const twoHundredOKCode = "200 OK"; -static const char * const twoHundredOneCreatedCode = "201 Created"; -static const char * const httpHeaderString = "HTTP/1.1"; - -/** - * Exception information after calling JSON operations - */ -struct jsonException { - const char *exception; - const char *javaClassName; - const char *message; -}; - -/** Print out the JSON exception information */ -static int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, - const char *fmt, va_list ap) -{ - char *javaClassName = NULL; - int excErrno = EINTERNAL, shouldPrint = 0; - if (!exc) { - fprintf(stderr, "printJsonExceptionV: the jsonException is NULL\n"); - return EINTERNAL; - } - javaClassName = strdup(exc->javaClassName); - if (!javaClassName) { - fprintf(stderr, "printJsonExceptionV: internal out of memory error\n"); - return EINTERNAL; - } - getExceptionInfo(javaClassName, noPrintFlags, &excErrno, &shouldPrint); - free(javaClassName); - - if (shouldPrint) { - vfprintf(stderr, fmt, ap); - fprintf(stderr, " error:\n"); - fprintf(stderr, "Exception: %s\nJavaClassName: %s\nMessage: %s\n", - exc->exception, exc->javaClassName, exc->message); - } - - free(exc); - return excErrno; -} - -/** - * Print out JSON exception information. - * - * @param exc The exception information to print and free - * @param noPrintFlags Flags which determine which exceptions we should NOT - * print. - * @param fmt Printf-style format list - * @param ... Printf-style varargs - * - * @return The POSIX error number associated with the exception - * object. - */ -static int printJsonException(struct jsonException *exc, int noPrintFlags, - const char *fmt, ...) -{ - va_list ap; - int ret = 0; - - va_start(ap, fmt); - ret = printJsonExceptionV(exc, noPrintFlags, fmt, ap); - va_end(ap); - return ret; -} - -/** Parse the exception information from JSON */ -static struct jsonException *parseJsonException(json_t *jobj) -{ - const char *key = NULL; - json_t *value = NULL; - struct jsonException *exception = NULL; - void *iter = NULL; - - exception = calloc(1, sizeof(*exception)); - if (!exception) { - return NULL; - } - - iter = json_object_iter(jobj); - while (iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "exception")) { - exception->exception = json_string_value(value); - } else if (!strcmp(key, "javaClassName")) { - exception->javaClassName = json_string_value(value); - } else if (!strcmp(key, "message")) { - exception->message = json_string_value(value); - } - - iter = json_object_iter_next(jobj, iter); - } - return exception; -} - -/** - * Parse the exception information which is presented in JSON - * - * @param content Exception information in JSON - * @return jsonException for printing out - */ -static struct jsonException *parseException(const char *content) -{ - json_error_t error; - size_t flags = 0; - const char *key = NULL; - json_t *value; - json_t *jobj; - struct jsonException *exception = NULL; - - if (!content) { - return NULL; - } - jobj = json_loads(content, flags, &error); - if (!jobj) { - fprintf(stderr, "JSon parsing error: on line %d: %s\n", - error.line, error.text); - return NULL; - } - void *iter = json_object_iter(jobj); - while(iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "RemoteException") && - json_typeof(value) == JSON_OBJECT) { - exception = parseJsonException(value); - break; - } - iter = json_object_iter_next(jobj, iter); - } - - json_decref(jobj); - return exception; -} - -/** - * Parse the response information which uses TRUE/FALSE - * to indicate whether the operation succeeded - * - * @param response Response information - * @return 0 to indicate success - */ -static int parseBoolean(const char *response) -{ - json_t *root, *value; - json_error_t error; - size_t flags = 0; - int result = 0; - - root = json_loads(response, flags, &error); - if (!root) { - fprintf(stderr, "JSon parsing error: on line %d: %s\n", - error.line, error.text); - return EIO; - } - void *iter = json_object_iter(root); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_TRUE) { - result = 0; - } else { - result = EIO; // FALSE means error in remote NN/DN - } - json_decref(root); - return result; -} - -int parseMKDIR(const char *response) -{ - return parseBoolean(response); -} - -int parseRENAME(const char *response) -{ - return parseBoolean(response); -} - -int parseDELETE(const char *response) -{ - return parseBoolean(response); -} - -int parseSETREPLICATION(const char *response) -{ - return parseBoolean(response); -} - -/** - * Check the header of response to see if it's 200 OK - * - * @param header Header information for checking - * @param content Stores exception information if there are errors - * @param operation Indicate the operation for exception printing - * @return 0 for success - */ -static int checkHeader(const char *header, const char *content, - const char *operation) -{ - char *result = NULL; - const char delims[] = ":"; - char *savepter; - int ret = 0; - - if (!header || strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOKCode)) || - !(result = strstr(header, "Content-Length"))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (%s)", operation); - } else { - ret = EIO; - } - return ret; - } - result = strtok_r(result, delims, &savepter); - result = strtok_r(NULL, delims, &savepter); - while (isspace(*result)) { - result++; - } - // Content-Length should be equal to 0, - // and the string should be "0\r\nServer" - if (strncmp(result, "0\r\n", 3)) { - ret = EIO; - } - return ret; -} - -int parseCHMOD(const char *header, const char *content) -{ - return checkHeader(header, content, "CHMOD"); -} - -int parseCHOWN(const char *header, const char *content) -{ - return checkHeader(header, content, "CHOWN"); -} - -int parseUTIMES(const char *header, const char *content) -{ - return checkHeader(header, content, "SETTIMES"); -} - -/** - * Check if the header contains correct information - * ("307 TEMPORARY_REDIRECT" and "Location") - * - * @param header Header for parsing - * @param content Contains exception information - * if the remote operation failed - * @param operation Specify the remote operation when printing out exception - * @return 0 for success - */ -static int checkRedirect(const char *header, - const char *content, const char *operation) -{ - const char *locTag = "Location"; - int ret = 0, offset = 0; - - // The header must start with "HTTP/1.1" - if (!header || strncmp(header, httpHeaderString, - strlen(httpHeaderString))) { - return EINVAL; - } - - offset += strlen(httpHeaderString); - while (isspace(header[offset])) { - offset++; - } - // Looking for "307 TEMPORARY_REDIRECT" in header - if (strncmp(header + offset, temporaryRedirectCode, - strlen(temporaryRedirectCode))) { - // Process possible exception information - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (%s)", operation); - } else { - ret = EIO; - } - return ret; - } - // Here we just simply check if header contains "Location" tag, - // detailed processing is in parseDnLoc - if (!(strstr(header, locTag))) { - ret = EIO; - } - return ret; -} - -int parseNnWRITE(const char *header, const char *content) -{ - return checkRedirect(header, content, "Write(NameNode)"); -} - -int parseNnAPPEND(const char *header, const char *content) -{ - return checkRedirect(header, content, "Append(NameNode)"); -} - -/** 0 for success , -1 for out of range, other values for error */ -int parseOPEN(const char *header, const char *content) -{ - int ret = 0, offset = 0; - - if (!header || strncmp(header, httpHeaderString, - strlen(httpHeaderString))) { - return EINVAL; - } - - offset += strlen(httpHeaderString); - while (isspace(header[offset])) { - offset++; - } - if (strncmp(header + offset, temporaryRedirectCode, - strlen(temporaryRedirectCode)) || - !strstr(header, twoHundredOKCode)) { - struct jsonException *exc = parseException(content); - if (exc) { - // If the exception is an IOException and it is because - // the offset is out of the range, do not print out the exception - if (!strcasecmp(exc->exception, "IOException") && - strstr(exc->message, "out of the range")) { - ret = -1; - } else { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (OPEN)"); - } - } else { - ret = EIO; - } - } - return ret; -} - -int parseDnLoc(char *content, char **dn) -{ - char *url = NULL, *dnLocation = NULL, *savepter, *tempContent; - const char *prefix = "Location: http://"; - const char *prefixToRemove = "Location: "; - const char *delims = "\r\n"; - - tempContent = strdup(content); - if (!tempContent) { - return ENOMEM; - } - - dnLocation = strtok_r(tempContent, delims, &savepter); - while (dnLocation && strncmp(dnLocation, "Location:", - strlen("Location:"))) { - dnLocation = strtok_r(NULL, delims, &savepter); - } - if (!dnLocation) { - return EIO; - } - - while (isspace(*dnLocation)) { - dnLocation++; - } - if (strncmp(dnLocation, prefix, strlen(prefix))) { - return EIO; - } - url = strdup(dnLocation + strlen(prefixToRemove)); - if (!url) { - return ENOMEM; - } - *dn = url; - return 0; -} - -int parseDnWRITE(const char *header, const char *content) -{ - int ret = 0; - if (header == NULL || header[0] == '\0' || - strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOneCreatedCode))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (WRITE(DataNode))"); - } else { - ret = EIO; - } - } - return ret; -} - -int parseDnAPPEND(const char *header, const char *content) -{ - int ret = 0; - - if (header == NULL || header[0] == '\0' || - strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOKCode))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (APPEND(DataNode))"); - } else { - ret = EIO; - } - } - return ret; -} - -/** - * Retrieve file status from the JSON object - * - * @param jobj JSON object for parsing, which contains - * file status information - * @param fileStat hdfsFileInfo handle to hold file status information - * @return 0 on success - */ -static int parseJsonForFileStatus(json_t *jobj, hdfsFileInfo *fileStat) -{ - const char *key, *tempstr; - json_t *value; - void *iter = NULL; - - iter = json_object_iter(jobj); - while (iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "accessTime")) { - // json field contains time in milliseconds, - // hdfsFileInfo is counted in seconds - fileStat->mLastAccess = json_integer_value(value) / 1000; - } else if (!strcmp(key, "blockSize")) { - fileStat->mBlockSize = json_integer_value(value); - } else if (!strcmp(key, "length")) { - fileStat->mSize = json_integer_value(value); - } else if (!strcmp(key, "modificationTime")) { - fileStat->mLastMod = json_integer_value(value) / 1000; - } else if (!strcmp(key, "replication")) { - fileStat->mReplication = json_integer_value(value); - } else if (!strcmp(key, "group")) { - fileStat->mGroup = strdup(json_string_value(value)); - if (!fileStat->mGroup) { - return ENOMEM; - } - } else if (!strcmp(key, "owner")) { - fileStat->mOwner = strdup(json_string_value(value)); - if (!fileStat->mOwner) { - return ENOMEM; - } - } else if (!strcmp(key, "pathSuffix")) { - fileStat->mName = strdup(json_string_value(value)); - if (!fileStat->mName) { - return ENOMEM; - } - } else if (!strcmp(key, "permission")) { - tempstr = json_string_value(value); - fileStat->mPermissions = (short) strtol(tempstr, NULL, 8); - } else if (!strcmp(key, "type")) { - tempstr = json_string_value(value); - if (!strcmp(tempstr, "DIRECTORY")) { - fileStat->mKind = kObjectKindDirectory; - } else { - fileStat->mKind = kObjectKindFile; - } - } - // Go to the next key-value pair in the json object - iter = json_object_iter_next(jobj, iter); - } - return 0; -} - -int parseGFS(const char *response, hdfsFileInfo *fileStat, int printError) -{ - int ret = 0, printFlag; - json_error_t error; - size_t flags = 0; - json_t *jobj, *value; - const char *key; - void *iter = NULL; - - if (!response || !fileStat) { - return EIO; - } - jobj = json_loads(response, flags, &error); - if (!jobj) { - fprintf(stderr, "error while parsing json: on line %d: %s\n", - error.line, error.text); - return EIO; - } - iter = json_object_iter(jobj); - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_OBJECT) { - if (!strcmp(key, "RemoteException")) { - struct jsonException *exception = parseJsonException(value); - if (exception) { - if (printError) { - printFlag = PRINT_EXC_ALL; - } else { - printFlag = NOPRINT_EXC_FILE_NOT_FOUND | - NOPRINT_EXC_ACCESS_CONTROL | - NOPRINT_EXC_PARENT_NOT_DIRECTORY; - } - ret = printJsonException(exception, printFlag, - "Calling WEBHDFS GETFILESTATUS"); - } else { - ret = EIO; - } - } else if (!strcmp(key, "FileStatus")) { - ret = parseJsonForFileStatus(value, fileStat); - } else { - ret = EIO; - } - - } else { - ret = EIO; - } - - json_decref(jobj); - return ret; -} - -/** - * Parse the JSON array. Called to parse the result of - * the LISTSTATUS operation. Thus each element of the JSON array is - * a JSON object with the information of a file entry contained - * in the folder. - * - * @param jobj The JSON array to be parsed - * @param fileStat The hdfsFileInfo handle used to - * store a group of file information - * @param numEntries Capture the number of files in the folder - * @return 0 for success - */ -static int parseJsonArrayForFileStatuses(json_t *jobj, hdfsFileInfo **fileStat, - int *numEntries) -{ - json_t *jvalue = NULL; - int i = 0, ret = 0, arraylen = 0; - hdfsFileInfo *fileInfo = NULL; - - arraylen = (int) json_array_size(jobj); - if (arraylen > 0) { - fileInfo = calloc(arraylen, sizeof(hdfsFileInfo)); - if (!fileInfo) { - return ENOMEM; - } - } - for (i = 0; i < arraylen; i++) { - //Getting the array element at position i - jvalue = json_array_get(jobj, i); - if (json_is_object(jvalue)) { - ret = parseJsonForFileStatus(jvalue, &fileInfo[i]); - if (ret) { - goto done; - } - } else { - ret = EIO; - goto done; - } - } -done: - if (ret) { - free(fileInfo); - } else { - *numEntries = arraylen; - *fileStat = fileInfo; - } - return ret; -} - -int parseLS(const char *response, hdfsFileInfo **fileStats, int *numOfEntries) -{ - int ret = 0; - json_error_t error; - size_t flags = 0; - json_t *jobj, *value; - const char *key; - void *iter = NULL; - - if (!response || response[0] == '\0' || !fileStats) { - return EIO; - } - jobj = json_loads(response, flags, &error); - if (!jobj) { - fprintf(stderr, "error while parsing json: on line %d: %s\n", - error.line, error.text); - return EIO; - } - - iter = json_object_iter(jobj); - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_OBJECT) { - if (!strcmp(key, "RemoteException")) { - struct jsonException *exception = parseJsonException(value); - if (exception) { - ret = printJsonException(exception, PRINT_EXC_ALL, - "Calling WEBHDFS GETFILESTATUS"); - } else { - ret = EIO; - } - } else if (!strcmp(key, "FileStatuses")) { - iter = json_object_iter(value); - value = json_object_iter_value(iter); - if (json_is_array(value)) { - ret = parseJsonArrayForFileStatuses(value, fileStats, - numOfEntries); - } else { - ret = EIO; - } - } else { - ret = EIO; - } - } else { - ret = EIO; - } - - json_decref(jobj); - return ret; -}