move experimental C++ LevelDB backend into Apache Marmotta main, and named the new module "ostrich" as an analogy to "kiwi"
Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/0ff22a0c Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/0ff22a0c Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/0ff22a0c Branch: refs/heads/develop Commit: 0ff22a0c3e086efdae4f25aa11aa27ff987c99bc Parents: ed387b9 Author: Sebastian Schaffert <[email protected]> Authored: Sat Dec 12 17:07:51 2015 +0100 Committer: Sebastian Schaffert <[email protected]> Committed: Sat Dec 12 17:07:51 2015 +0100 ---------------------------------------------------------------------- launchers/marmotta-webapp/pom.xml | 22 + libraries/ostrich/backend/CMakeLists.txt | 31 + libraries/ostrich/backend/README.md | 87 + libraries/ostrich/backend/client/CMakeLists.txt | 8 + libraries/ostrich/backend/client/client.cc | 275 + .../ostrich/backend/cmake/FindGFlags.cmake | 48 + libraries/ostrich/backend/cmake/FindGLog.cmake | 18 + libraries/ostrich/backend/cmake/FindGRPC.cmake | 67 + .../ostrich/backend/cmake/FindLevelDB.cmake | 18 + .../ostrich/backend/cmake/FindRAPTOR.cmake | 103 + .../ostrich/backend/cmake/FindRasqal.cmake | 99 + .../ostrich/backend/cmake/FindTcmalloc.cmake | 39 + libraries/ostrich/backend/model/CMakeLists.txt | 6 + libraries/ostrich/backend/model/model.proto | 77 + libraries/ostrich/backend/model/rdf_model.cc | 348 + libraries/ostrich/backend/model/rdf_model.h | 709 + .../ostrich/backend/model/rdf_operators.cc | 61 + libraries/ostrich/backend/model/rdf_operators.h | 240 + libraries/ostrich/backend/parser/CMakeLists.txt | 4 + libraries/ostrich/backend/parser/rdf_parser.cc | 175 + libraries/ostrich/backend/parser/rdf_parser.h | 87 + .../ostrich/backend/persistence/CMakeLists.txt | 10 + .../backend/persistence/leveldb_persistence.cc | 685 + .../backend/persistence/leveldb_persistence.h | 185 + .../backend/persistence/leveldb_server.cc | 73 + .../backend/persistence/leveldb_service.cc | 254 + .../backend/persistence/leveldb_service.h | 120 + .../backend/persistence/leveldb_sparql.cc | 114 + .../backend/persistence/leveldb_sparql.h | 52 + .../ostrich/backend/serializer/CMakeLists.txt | 4 + .../ostrich/backend/serializer/serializer.cc | 49 + .../ostrich/backend/serializer/serializer.h | 54 + .../backend/serializer/serializer_base.cc | 64 + .../backend/serializer/serializer_base.h | 104 + .../backend/serializer/serializer_proto.cc | 54 + .../backend/serializer/serializer_proto.h | 50 + .../backend/serializer/serializer_raptor.cc | 266 + .../backend/serializer/serializer_raptor.h | 55 + .../ostrich/backend/service/CMakeLists.txt | 9 + libraries/ostrich/backend/service/sail.proto | 102 + libraries/ostrich/backend/service/sparql.proto | 45 + .../ostrich/backend/sharding/CMakeLists.txt | 10 + libraries/ostrich/backend/sharding/server.cc | 66 + libraries/ostrich/backend/sharding/sharding.cc | 335 + libraries/ostrich/backend/sharding/sharding.h | 174 + libraries/ostrich/backend/sparql/CMakeLists.txt | 7 + .../ostrich/backend/sparql/rasqal_adapter.cc | 299 + .../ostrich/backend/sparql/rasqal_adapter.h | 107 + .../ostrich/backend/sparql/rasqal_model.cc | 193 + libraries/ostrich/backend/sparql/rasqal_model.h | 74 + libraries/ostrich/backend/test/CMakeLists.txt | 12 + libraries/ostrich/backend/test/SparqlTest.cc | 266 + libraries/ostrich/backend/test/StatementTest.cc | 135 + libraries/ostrich/backend/test/gtest-all.cc | 9592 ++++++++ libraries/ostrich/backend/test/gtest.h | 20061 +++++++++++++++++ libraries/ostrich/backend/test/main.cc | 11 + libraries/ostrich/backend/util/CMakeLists.txt | 3 + libraries/ostrich/backend/util/iterator.h | 131 + libraries/ostrich/backend/util/murmur3.cc | 313 + libraries/ostrich/backend/util/murmur3.h | 18 + libraries/ostrich/backend/util/split.cc | 40 + libraries/ostrich/backend/util/split.h | 38 + libraries/ostrich/client/pom.xml | 234 + .../ostrich/sail/ClosableResponseStream.java | 163 + .../marmotta/ostrich/sail/OstrichSail.java | 87 + .../ostrich/sail/OstrichSailConnection.java | 529 + .../ostrich/sail/OstrichValueFactory.java | 260 + .../ostrich/sail/test/CMarmottaSailTest.java | 74 + .../ostrich/sail/test/TestSailConnection.java | 80 + libraries/ostrich/model/pom.xml | 219 + .../marmotta/ostrich/model/ProtoBNode.java | 79 + .../ostrich/model/ProtoDatatypeLiteral.java | 106 + .../ostrich/model/ProtoLiteralBase.java | 187 + .../marmotta/ostrich/model/ProtoNamespace.java | 86 + .../marmotta/ostrich/model/ProtoStatement.java | 221 + .../ostrich/model/ProtoStringLiteral.java | 101 + .../apache/marmotta/ostrich/model/ProtoURI.java | 113 + .../ostrich/model/test/StatementTest.java | 68 + .../marmotta/ostrich/model/test/URITest.java | 57 + .../model/src/test/resources/logback.xml | 28 + libraries/ostrich/pom.xml | 50 + libraries/pom.xml | 9 + .../backends/marmotta-backend-ostrich/pom.xml | 175 + .../backend/ostrich/OstrichProvider.java | 94 + .../src/main/resources/META-INF/beans.xml | 28 + .../main/resources/config-defaults.properties | 24 + .../resources/config-descriptions.properties | 26 + .../src/main/resources/kiwi-module.properties | 38 + .../src/main/resources/web/admin/about.html | 36 + .../main/resources/web/admin/configuration.html | 52 + platform/backends/pom.xml | 6 + 91 files changed, 39986 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/launchers/marmotta-webapp/pom.xml ---------------------------------------------------------------------- diff --git a/launchers/marmotta-webapp/pom.xml b/launchers/marmotta-webapp/pom.xml index c9d0cd6..e02e44a 100644 --- a/launchers/marmotta-webapp/pom.xml +++ b/launchers/marmotta-webapp/pom.xml @@ -415,6 +415,28 @@ </dependencies> </profile> + <profile> + <id>ostrich</id> + <activation> + <property> + <name>marmotta.backend</name> + <value>ostrich</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.marmotta</groupId> + <artifactId>marmotta-backend-ostrich</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.marmotta</groupId> + <artifactId>marmotta-ldcache-file</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> + <!-- Caching Backends for KiWi --> http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt new file mode 100644 index 0000000..d8232b7 --- /dev/null +++ b/libraries/ostrich/backend/CMakeLists.txt @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.0) +project(Marmotta) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g") +set(PROTOBUF_IMPORT_DIRS "${CMAKE_SOURCE_DIR}/model") +set(USE_TCMALLOC TRUE) + +find_package (Threads REQUIRED) +find_package (RAPTOR REQUIRED) +find_package (Rasqal REQUIRED) +find_package (GFlags REQUIRED) +find_package (Protobuf REQUIRED) +find_package (GRPC REQUIRED) +find_package (LevelDB REQUIRED) +find_package (GLog REQUIRED) +find_package (Tcmalloc) + +add_definitions(-DNDEBUG) + +add_subdirectory(util) +add_subdirectory(model) +add_subdirectory(sparql) +add_subdirectory(service) +add_subdirectory(parser) +add_subdirectory(serializer) +add_subdirectory(persistence) +add_subdirectory(sharding) +add_subdirectory(client) +add_subdirectory(test) + http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/README.md ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/README.md b/libraries/ostrich/backend/README.md new file mode 100644 index 0000000..047a968 --- /dev/null +++ b/libraries/ostrich/backend/README.md @@ -0,0 +1,87 @@ +# Apache Marmotta LevelDB/C++ Backend + +This repository implements an experimental high-performance backend for Apache Marmotta +using LevelDB as storage and gRPC as communication channel between the Java frontend +and the C++ backend. + +If it proves to be useful, the repository will eventually be merged into the main +development branch of Apache Marmotta + +## Dependencies (C++) + +To compile the C++ backend, you need to have the following dependencies installed: + + * libraptor (used for parsing/serializing in C++) + * librasqal (used for server-side SPARQL evaluation) + * libglog (logging) + * libgflags (command line arguments) + * libleveldb (database backend) + * libgrpc (gRPC runtime) + * libprotobuf (messaging, data model) + +With the exception of libgrpc and libprotobuf, all libraries are available in Linux repositories. +Debian: + + apt-get install libraptor2-dev librasqal3-dev libgoogle-glog-dev libgflags-dev libleveldb-dev + +The backend uses the new Proto 3 format and the gRPC SDK. These need to be installed separately, +please follow the instructions at [https://github.com/grpc/grpc](https://github.com/grpc/grpc/blob/master/INSTALL). + + +## Compilation (C++) + +The backend uses cmake to compile the modules. Create a new directory `build`, run cmake, and run make: + + mkdir build && cd build + cmake .. + make + +## Compilation (Java) + +The frontend is compiled with Maven and depends on many Apache Marmotta modules to work. Build it with + + cd java + mvn clean install + +## Running C++ Backend + +Start the backend from the cmake build directory as follows: + + ./service/marmotta_persistence -db /path/to/database -port 10000 + +The binary accepts many different options. Please see `--help` for details. + +## Running Sharding + +The repository contains an experimental implementation of a sharding server that proxies and +distributes requests based on a hash calculation over statements. In heavy load environments, +this is potentially much faster than running a single persistence backend. The setup requires +several persistence backends (shards) and a sharding proxy. To experiment, you can start these +on the same machine as follows: + + ./service/marmotta_persistence -db /path/to/shard1 -port 10001 + ./service/marmotta_persistence -db /path/to/shard2 -port 10002 + ./sharding/marmotta_sharding --port 10000 --backends localhost:10001,localhost:10002 + +You can then access the sharding server through Marmotta like the persistence server. Running all instances +on the same host is only useful for testing. In production environments, you would of course run all three +(or more) instances on different hosts. Note that the number and order of backends should not change once +data has been imported, because otherwise the hashing algorithm will do the wrong thing. + +## Running Apache Marmotta + +A preconfigured version of Apache Marmotta is available in `java/webapp`. It connects to +`localhost:10000` by default and can be started with: + + mvn tomcat7:run + +Afterwards, point your browser to `localhost:8080`. + +## Command Line Client + +A C++ command line client is available for very fast bulk imports and simple queries. To import +a large turtle file, run: + + ./client/marmotta_client --format=turtle import file.ttl + +The client connects by default to `localhost:10000` (change with `--host` and `--port` flags). http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/client/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/CMakeLists.txt b/libraries/ostrich/backend/client/CMakeLists.txt new file mode 100644 index 0000000..88ad11a --- /dev/null +++ b/libraries/ostrich/backend/client/CMakeLists.txt @@ -0,0 +1,8 @@ +include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_BINARY_DIR}/../model) + +add_executable(marmotta_client client.cc) +target_link_libraries(marmotta_client + marmotta_model marmotta_service marmotta_parser marmotta_serializer + ${GFLAGS_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES} ${GRPC_LIBRARIES}) + http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/client/client.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/client/client.cc b/libraries/ostrich/backend/client/client.cc new file mode 100644 index 0000000..16c9022 --- /dev/null +++ b/libraries/ostrich/backend/client/client.cc @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <fstream> + +#include <grpc/grpc.h> +#include <grpc++/channel.h> +#include <grpc++/client_context.h> +#include <grpc++/create_channel.h> +#include <grpc++/security/credentials.h> +#include <grpc++/support/sync_stream.h> + +#include <google/protobuf/text_format.h> +#include <google/protobuf/empty.pb.h> +#include <google/protobuf/wrappers.pb.h> + +#include <gflags/gflags.h> + +#include "model/rdf_model.h" +#include "parser/rdf_parser.h" +#include "serializer/serializer.h" +#include "service/sail.pb.h" +#include "service/sail.grpc.pb.h" +#include "service/sparql.pb.h" +#include "service/sparql.grpc.pb.h" + + +using grpc::Channel; +using grpc::ClientContext; +using grpc::ClientReader; +using grpc::ClientReaderWriter; +using grpc::ClientWriter; +using grpc::Status; +using google::protobuf::TextFormat; + +using namespace marmotta; +namespace svc = marmotta::service::proto; +namespace spq = marmotta::sparql::proto; + +// A STL iterator wrapper around a client reader. +template <class T, class Proto> +class ClientReaderIterator : public util::CloseableIterator<T> { + public: + ClientReaderIterator() : finished(true) { } + + ClientReaderIterator(ClientReader<Proto>* r) : reader(r), finished(false) { + // Immediately move to first element. + operator++(); + } + + ClientReaderIterator& operator++() override { + if (!finished) { + finished = !reader->Read(&buffer); + current = T(buffer); + if (finished) { + reader->Finish(); + } + } + return *this; + } + + T& operator*() override { + return current; + } + + T* operator->() override { + return ¤t; + } + + bool hasNext() override { + return !finished; + } + + private: + ClientReader<Proto>* reader; + Proto buffer; + T current; + bool finished; +}; + +typedef ClientReaderIterator<rdf::Statement, rdf::proto::Statement> StatementReader; +typedef ClientReaderIterator<rdf::Namespace, rdf::proto::Namespace> NamespaceReader; + +class MarmottaClient { + public: + MarmottaClient(const std::string& server) + : stub_(svc::SailService::NewStub( + grpc::CreateChannel(server, grpc::InsecureChannelCredentials()))), + sparql_(spq::SparqlService::NewStub( + grpc::CreateChannel(server, grpc::InsecureChannelCredentials()))){} + + void importDataset(std::istream& in, parser::Format format) { + ClientContext nscontext, stmtcontext; + + google::protobuf::Int64Value nsstats; + google::protobuf::Int64Value stmtstats; + + std::unique_ptr<ClientWriter<rdf::proto::Namespace> > nswriter( + stub_->AddNamespaces(&nscontext, &nsstats)); + std::unique_ptr<ClientWriter<rdf::proto::Statement> > stmtwriter( + stub_->AddStatements(&stmtcontext, &stmtstats)); + + parser::Parser p("http://www.example.com";, format); + p.setStatementHandler([&stmtwriter](const rdf::Statement& stmt) { + stmtwriter->Write(stmt.getMessage()); + }); + p.setNamespaceHandler([&nswriter](const rdf::Namespace& ns) { + nswriter->Write(ns.getMessage()); + }); + p.parse(in); + + stmtwriter->WritesDone(); + nswriter->WritesDone(); + + Status nsst = nswriter->Finish(); + Status stmtst = stmtwriter->Finish(); + + if (nsst.ok() && stmtst.ok()) { + std::cout << "Added " << nsstats.value() << " namespaces and " + << stmtstats.value() << " statements" << std::endl; + } else { + std::cout << "Failed writing data to server: " << stmtst.error_message() << std::endl; + } + } + + + void patternQuery(const rdf::Statement &pattern, std::ostream &out, serializer::Format format) { + ClientContext context; + + std::unique_ptr<ClientReader<rdf::proto::Statement> > reader( + stub_->GetStatements(&context, pattern.getMessage())); + + serializer::Serializer serializer("http://www.example.com";, format); + serializer.serialize(StatementReader(reader.get()), out); + } + + void patternDelete(const rdf::Statement &pattern) { + ClientContext context; + google::protobuf::Int64Value result; + + Status status = stub_->RemoveStatements(&context, pattern.getMessage(), &result); + if (status.ok()) { + std::cout << "Deleted " << result.value() << " statements." << std::endl; + } else { + std::cerr << "Failed deleting statements: " << status.error_message() << std::endl; + } + } + + void tupleQuery(const std::string& query, std::ostream &out) { + ClientContext context; + spq::SparqlRequest request; + request.set_query(query); + + std::unique_ptr<ClientReader<spq::SparqlResponse>> reader( + sparql_->TupleQuery(&context, request)); + + auto out_ = new google::protobuf::io::OstreamOutputStream(&out); + spq::SparqlResponse result; + while (reader->Read(&result)) { + TextFormat::Print(result, dynamic_cast<google::protobuf::io::ZeroCopyOutputStream*>(out_)); + } + delete out_; + } + + void listNamespaces(std::ostream &out) { + ClientContext context; + + google::protobuf::Empty pattern; + + std::unique_ptr<ClientReader<rdf::proto::Namespace> > reader( + stub_->GetNamespaces(&context, pattern)); + + NamespaceReader it(reader.get()); + for (; it.hasNext(); ++it) { + out << (*it).getPrefix() << " = " << (*it).getUri() << std::endl; + } + } + + int64_t size(const svc::ContextRequest& r) { + ClientContext context; + google::protobuf::Int64Value result; + + Status status = stub_->Size(&context, r, &result); + if (status.ok()) { + return result.value(); + } else { + return -1; + } + } + private: + std::unique_ptr<svc::SailService::Stub> stub_; + std::unique_ptr<spq::SparqlService::Stub> sparql_; +}; + + +DEFINE_string(format, "rdfxml", "RDF format to use for parsing/serializing."); +DEFINE_string(host, "localhost", "Address/name of server to access."); +DEFINE_string(port, "10000", "Port of server to access."); +DEFINE_string(output, "", "File to write result to."); + +int main(int argc, char** argv) { + GOOGLE_PROTOBUF_VERIFY_VERSION; + + google::ParseCommandLineFlags(&argc, &argv, true); + + MarmottaClient client(FLAGS_host + ":" + FLAGS_port); + + if ("import" == std::string(argv[1])) { + std::ifstream in(argv[2]); + std::cout << "Importing " << argv[2] << " ... " << std::endl; + client.importDataset(in, parser::FormatFromString(FLAGS_format)); + std::cout << "Finished!" << std::endl; + } + + if ("select" == std::string(argv[1])) { + rdf::proto::Statement query; + TextFormat::ParseFromString(argv[2], &query); + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.patternQuery(rdf::Statement(query), out, serializer::FormatFromString(FLAGS_format)); + } else { + client.patternQuery(rdf::Statement(query), std::cout, serializer::FormatFromString(FLAGS_format)); + } + } + + if ("sparql" == std::string(argv[1])) { + std::string query = argv[2]; + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.tupleQuery(query, out); + } else { + client.tupleQuery(query, std::cout); + } + } + + if ("delete" == std::string(argv[1])) { + rdf::proto::Statement query; + TextFormat::ParseFromString(argv[2], &query); + client.patternDelete(rdf::Statement(query)); + } + + if ("size" == std::string(argv[1])) { + svc::ContextRequest query; + TextFormat::ParseFromString(argv[2], &query); + std::cout << "Size: " << client.size(query) << std::endl; + } + + + if ("namespaces" == std::string(argv[1])) { + if (FLAGS_output != "") { + std::ofstream out(FLAGS_output); + client.listNamespaces(out); + } else { + client.listNamespaces(std::cout); + } + } + + google::protobuf::ShutdownProtobufLibrary(); + + return 0; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindGFlags.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindGFlags.cmake b/libraries/ostrich/backend/cmake/FindGFlags.cmake new file mode 100644 index 0000000..2d44a8c --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindGFlags.cmake @@ -0,0 +1,48 @@ +# - Try to find GFLAGS +# +# The following variables are optionally searched for defaults +# GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found +# +# The following are set after configuration is done: +# GFLAGS_FOUND +# GFLAGS_INCLUDE_DIRS +# GFLAGS_LIBRARIES +# GFLAGS_LIBRARYRARY_DIRS + +include(FindPackageHandleStandardArgs) + +set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") + +# We are testing only a couple of files in the include directories +if(WIN32) + find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h + PATHS ${GFLAGS_ROOT_DIR}/src/windows) +else() + find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h + PATHS ${GFLAGS_ROOT_DIR}) +endif() + +if(MSVC) + find_library(GFLAGS_LIBRARY_RELEASE + NAMES libgflags + PATHS ${GFLAGS_ROOT_DIR} + PATH_SUFFIXES Release) + + find_library(GFLAGS_LIBRARY_DEBUG + NAMES libgflags-debug + PATHS ${GFLAGS_ROOT_DIR} + PATH_SUFFIXES Debug) + + set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) +else() + find_library(GFLAGS_LIBRARY gflags) +endif() + +find_package_handle_standard_args(GFLAGS DEFAULT_MSG + GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) + + +if(GFLAGS_FOUND) + set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) + set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) +endif() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindGLog.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindGLog.cmake b/libraries/ostrich/backend/cmake/FindGLog.cmake new file mode 100644 index 0000000..3d09d06 --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindGLog.cmake @@ -0,0 +1,18 @@ +# Find Google Logging + +find_path(GLOG_INCLUDE_PATH NAMES glog/logging.h) +find_library(GLOG_LIBRARY NAMES glog) + +if(GLOG_INCLUDE_PATH AND GLOG_LIBRARY) + set(GLOG_FOUND TRUE) +endif(GLOG_INCLUDE_PATH AND GLOG_LIBRARY) + +if(GLOG_FOUND) + if(NOT GLOG_FIND_QUIETLY) + message(STATUS "Found GLOG: ${GLOG_LIBRARY}; includes - ${GLOG_INCLUDE_PATH}") + endif(NOT GLOG_FIND_QUIETLY) +else(GLOG_FOUND) + if(GLOG_FIND_REQUIRED) + message(FATAL_ERROR "Could not find GLOG library.") + endif(GLOG_FIND_REQUIRED) +endif(GLOG_FOUND) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindGRPC.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindGRPC.cmake b/libraries/ostrich/backend/cmake/FindGRPC.cmake new file mode 100644 index 0000000..d5aa718 --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindGRPC.cmake @@ -0,0 +1,67 @@ +find_program(GRPC_CPP_PLUGIN grpc_cpp_plugin) # Get full path to plugin + +find_library(GRPC_LIBRARY NAMES grpc) +find_library(GRPCPP_LIBRARY NAMES grpc++) +find_library(GPR_LIBRARY NAMES gpr) +set(GRPC_LIBRARIES ${GRPCPP_LIBRARY} ${GRPC_LIBRARY} ${GPR_LIBRARY}) +if(GRPC_LIBRARIES) + message(STATUS "Found GRPC: ${GRPC_LIBRARIES}; plugin - ${GRPC_CPP_PLUGIN}") +endif() + + +function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) # This variable is common for all types of output. + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS) + foreach(DIR ${PROTOBUF_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS --grpc_out=${CMAKE_CURRENT_BINARY_DIR} + --plugin=protoc-gen-grpc=${GRPC_CPP_PLUGIN} + ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} + COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}" + VERBATIM) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) +endfunction() + http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindLevelDB.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindLevelDB.cmake b/libraries/ostrich/backend/cmake/FindLevelDB.cmake new file mode 100644 index 0000000..db99bf9 --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindLevelDB.cmake @@ -0,0 +1,18 @@ +# Find libleveldb.a - key/value storage system + +find_path(LevelDB_INCLUDE_PATH NAMES leveldb/db.h) +find_library(LevelDB_LIBRARY NAMES leveldb) + +if(LevelDB_INCLUDE_PATH AND LevelDB_LIBRARY) + set(LevelDB_FOUND TRUE) +endif(LevelDB_INCLUDE_PATH AND LevelDB_LIBRARY) + +if(LevelDB_FOUND) + if(NOT LevelDB_FIND_QUIETLY) + message(STATUS "Found LevelDB: ${LevelDB_LIBRARY}; includes - ${LevelDB_INCLUDE_PATH}") + endif(NOT LevelDB_FIND_QUIETLY) +else(LevelDB_FOUND) + if(LevelDB_FIND_REQUIRED) + message(FATAL_ERROR "Could not find leveldb library.") + endif(LevelDB_FIND_REQUIRED) +endif(LevelDB_FOUND) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindRAPTOR.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindRAPTOR.cmake b/libraries/ostrich/backend/cmake/FindRAPTOR.cmake new file mode 100644 index 0000000..ef12f1b --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindRAPTOR.cmake @@ -0,0 +1,103 @@ +# - Try to find the Raptor RDF parsing library (http://librdf.org/raptor/) +# Once done this will define +# +# RAPTOR_FOUND - system has Raptor +# RAPTOR_LIBRARIES - Link these to use Raptor +# RAPTOR_INCLUDE_DIR - Include directory for using Raptor +# RAPTOR_DEFINITIONS - Compiler switches required for using Raptor +# +# Capabilities +# RAPTOR_HAVE_TRIG - Set if raptor has TRIG + +# (c) 2007-2011 Sebastian Trueg <[email protected]> +# (c) 2011 Artem Serebriyskiy <[email protected]> +# (c) 2011 Michael Jansen <[email protected]> +# +# Based on FindFontconfig Copyright (c) 2006,2007 Laurent Montel, <[email protected]> +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + + +MACRO ( FIND_RAPTOR ) + +ENDMACRO () + + + +# Check if we have cached results in case the last round was successful. +if ( NOT( RAPTOR_INCLUDE_DIR AND RAPTOR_LIBRARIES ) OR NOT RAPTOR_FOUND ) + + set( RAPTOR_LDFLAGS ) + + find_package(PkgConfig) + + if ( NOT WIN32 ) + pkg_check_modules(PC_RAPTOR QUIET raptor) + if ( PC_RAPTOR_FOUND ) + set(RAPTOR_DEFINITIONS ${PC_RAPTOR_CFLAGS_OTHER}) + set(RAPTOR_VERSION ${PC_RAPTOR_VERSION} CACHE STRING "Raptor Version found" ) + string( REGEX REPLACE "^.*-lraptor;" "" RAPTOR_LDFLAGS "${PC_RAPTOR_STATIC_LDFLAGS}") + endif () + endif () + + find_path(RAPTOR_INCLUDE_DIR + NAMES raptor2.h raptor2/raptor2.h raptor.h raptor/raptor.h + PATHS $ENV{RAPTOR_DIR}/include + $ENV{RAPTOR_DIR} + ~/Library/Frameworks + /Library/Frameworks + /usr/local/include + /usr/include/ + /sw/include # Fink + /opt/local/include # MacPorts + /opt/csw/include # Blastwave + /usr/local/opt/raptor/include # brew + /opt/include + /usr/freeware/include + + ) + + + find_library(RAPTOR_LIBRARY + NAMES raptor raptor2 + PATHS $ENV{RAPTOR_DIR}/lib + $ENV{RAPTOR_DIR}/lib-dbg + $ENV{RAPTOR_DIR} + ~/Library/Frameworks + /Library/Frameworks + /usr/local/lib + /usr/local/lib64 + /usr/lib + /usr/lib64 + /sw/lib # Fink + /opt/local/lib # MacPorts + /opt/csw/lib # Blastwave + /usr/local/opt/raptor/lib # brew + /opt/lib + /usr/freeware/lib64 + ) + + if ( RAPTOR_LDFLAGS ) + set( RAPTOR_LIBRARY ${RAPTOR_LIBRARY} ${RAPTOR_LDFLAGS} ) + endif () + + mark_as_advanced(RAPTOR_INCLUDE_DIR RAPTOR_LIBRARY) + +endif () # Check for cached values + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args( + Raptor + VERSION_VAR RAPTOR_VERSION + REQUIRED_VARS RAPTOR_LIBRARY RAPTOR_INCLUDE_DIR) + +mark_as_advanced(RAPTOR_VERSION) + +if (NOT RAPTOR_FOUND AND Raptor_FIND_VERSION_MAJOR EQUAL "2" AND NOT Raptor_FIND_QUIET ) + pkg_check_modules(PC_RAPTOR QUIET raptor) + if (PC_RAPTOR_FOUND) + message( STATUS "You have raptor1 version ${PC_RAPTOR_VERSION} installed. Please update." ) + endif () +endif () \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindRasqal.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindRasqal.cmake b/libraries/ostrich/backend/cmake/FindRasqal.cmake new file mode 100644 index 0000000..e80ab34 --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindRasqal.cmake @@ -0,0 +1,99 @@ +# - Try to find the Rasqal rdf query library (http://librdf.org/rasqal/) +# Once done this will define +# +# RASQAL_FOUND - system has Rasqal +# RASQAL_LIBRARIES - Link these to use RASQAL +# RASQAL_INCLUDE_DIR - The include directory for using rasqal +# RASQAL_DEFINITIONS - Compiler switches required for using RASQAL +# RASQAL_VERSION - The rasqal version string + +# (c) 2007-2009 Sebastian Trueg <[email protected]> +# +# Based on FindFontconfig Copyright (c) 2006,2007 Laurent Montel, <[email protected]> +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + + +if(WINCE) + FIND_PROGRAM( + RASQAL_CONFIG + NAMES rasqal-config + PATHS ${HOST_BINDIR} NO_DEFAULT_PATH + ) +else(WINCE) + FIND_PROGRAM( + RASQAL_CONFIG + NAMES rasqal-config + ) +endif(WINCE) + + if(RASQAL_CONFIG) + EXECUTE_PROCESS( + COMMAND ${RASQAL_CONFIG} --version + OUTPUT_VARIABLE RASQAL_VERSION + ) + if(RASQAL_VERSION) + STRING(REPLACE "\n" "" RASQAL_VERSION ${RASQAL_VERSION}) + + # extract include paths from rasqal-config + EXECUTE_PROCESS( + COMMAND ${RASQAL_CONFIG} --cflags + OUTPUT_VARIABLE rasqal_CFLAGS_ARGS) + STRING( REPLACE " " ";" rasqal_CFLAGS_ARGS ${rasqal_CFLAGS_ARGS} ) + FOREACH( _ARG ${rasqal_CFLAGS_ARGS} ) + IF(${_ARG} MATCHES "^-I") + STRING(REGEX REPLACE "^-I" "" _ARG ${_ARG}) + STRING( REPLACE "\n" "" _ARG ${_ARG} ) + LIST(APPEND rasqal_INCLUDE_DIRS ${_ARG}) + ENDIF(${_ARG} MATCHES "^-I") + ENDFOREACH(_ARG) + + # extract lib paths from rasqal-config + EXECUTE_PROCESS( + COMMAND ${RASQAL_CONFIG} --libs + OUTPUT_VARIABLE rasqal_CFLAGS_ARGS) + STRING( REPLACE " " ";" rasqal_CFLAGS_ARGS ${rasqal_CFLAGS_ARGS} ) + FOREACH( _ARG ${rasqal_CFLAGS_ARGS} ) + IF(${_ARG} MATCHES "^-L") + STRING(REGEX REPLACE "^-L" "" _ARG ${_ARG}) + LIST(APPEND rasqal_LIBRARY_DIRS ${_ARG}) + ENDIF(${_ARG} MATCHES "^-L") + ENDFOREACH(_ARG) + endif(RASQAL_VERSION) + endif(RASQAL_CONFIG) + + find_path(RASQAL_INCLUDE_DIR rasqal.h + PATHS + ${redland_INCLUDE_DIRS} + ${rasqal_INCLUDE_DIRS} + /usr/X11/include + PATH_SUFFIXES redland rasqal + ) + + find_library(RASQAL_LIBRARIES NAMES rasqal librasqal + PATHS + ${rasqal_LIBRARY_DIRS} + ) + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args( + Rasqal + VERSION_VAR RASQAL_VERSION + REQUIRED_VARS RASQAL_LIBRARIES RASQAL_INCLUDE_DIR) + + if (RASQAL_FOUND) + set(RASQAL_DEFINITIONS ${rasqal_CFLAGS}) + if (NOT Rasqal_FIND_QUIETLY) + message(STATUS "Found Rasqal ${RASQAL_VERSION}: libs - ${RASQAL_LIBRARIES}; includes - ${RASQAL_INCLUDE_DIR}") + endif (NOT Rasqal_FIND_QUIETLY) + else (RASQAL_FOUND) + if (Rasqal_FIND_REQUIRED) + message(FATAL_ERROR "Could NOT find Rasqal") + endif (Rasqal_FIND_REQUIRED) + endif (RASQAL_FOUND) + + +mark_as_advanced(RASQAL_INCLUDE_DIR_TMP + RASQAL_INCLUDE_DIR + RASQAL_LIBRARIES) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/cmake/FindTcmalloc.cmake ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/cmake/FindTcmalloc.cmake b/libraries/ostrich/backend/cmake/FindTcmalloc.cmake new file mode 100644 index 0000000..eb89d61 --- /dev/null +++ b/libraries/ostrich/backend/cmake/FindTcmalloc.cmake @@ -0,0 +1,39 @@ +# - Find Tcmalloc +# Find the native Tcmalloc includes and library +# +# Tcmalloc_INCLUDE_DIR - where to find Tcmalloc.h, etc. +# Tcmalloc_LIBRARIES - List of libraries when using Tcmalloc. +# Tcmalloc_FOUND - True if Tcmalloc found. + +find_path(Tcmalloc_INCLUDE_DIR google/tcmalloc.h) + +if (USE_TCMALLOC) + set(Tcmalloc_NAMES tcmalloc_and_profiler) +else () + set(Tcmalloc_NAMES tcmalloc_minimal tcmalloc) +endif () + +find_library(Tcmalloc_LIBRARY NAMES ${Tcmalloc_NAMES}) + +if (Tcmalloc_INCLUDE_DIR AND Tcmalloc_LIBRARY) + set(Tcmalloc_FOUND TRUE) + set( Tcmalloc_LIBRARIES ${Tcmalloc_LIBRARY} ) +else () + set(Tcmalloc_FOUND FALSE) + set( Tcmalloc_LIBRARIES ) +endif () + +if (Tcmalloc_FOUND) + message(STATUS "Found Tcmalloc: ${Tcmalloc_LIBRARY}") +else () + message(STATUS "Not Found Tcmalloc") + if (Tcmalloc_FIND_REQUIRED) + message(STATUS "Looked for Tcmalloc libraries named ${Tcmalloc_NAMES}.") + message(FATAL_ERROR "Could NOT find Tcmalloc library") + endif () +endif () + +mark_as_advanced( + Tcmalloc_LIBRARY + Tcmalloc_INCLUDE_DIR + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/CMakeLists.txt b/libraries/ostrich/backend/model/CMakeLists.txt new file mode 100644 index 0000000..473e6c8 --- /dev/null +++ b/libraries/ostrich/backend/model/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB ProtoFiles "${CMAKE_CURRENT_SOURCE_DIR}/*.proto") +PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS ${ProtoFiles}) +include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/..) + +add_library(marmotta_model rdf_model.cc rdf_model.h ${PROTO_SRCS} ${PROTO_HDRS} rdf_operators.h rdf_operators.cc) +target_link_libraries(marmotta_model ${CMAKE_THREAD_LIBS_INIT} ${PROTOBUF_LIBRARIES}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/model.proto ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/model.proto b/libraries/ostrich/backend/model/model.proto new file mode 100644 index 0000000..2076303 --- /dev/null +++ b/libraries/ostrich/backend/model/model.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package marmotta.rdf.proto; + +option java_package = "org.apache.marmotta.ostrich.model.proto"; + +// Namespaces consist of a prefix (short string used as replacement of a +// URI prefix) and a uri (the URI to replace by a prefix) +message Namespace { + string prefix = 1; + string uri = 2; +} + +// URI resources have a single required field, the uri they are pointing to. +message URI { + string uri = 1; +} + +// BNodes/anonymous nodes have a single required field, the node ID. +message BNode { + string id = 1; +} + +// Resources are either URIs or BNodes. +message Resource { + oneof Resources { + URI uri = 1; + BNode bnode = 2; + } +} + +// A string literal has string content and an optional language specification. +// At least content is required. +message StringLiteral { + string content = 1; + string language = 2; +} + +// A datatype literal has string content of a specific datatype and a URI +// identifying that datatype (typically in XSD namespace). Both fields are +// required. +message DatatypeLiteral { + string content = 1; + URI datatype = 2; +} + +// A literal is either a string literal with optional language or a +// datatype literal with required content and datatype. +message Literal { + oneof Literals { + StringLiteral stringliteral = 1; + DatatypeLiteral dataliteral = 2; + } +} + +// Values can be resources or literals +message Value { + oneof Values { + Resource resource = 1; + Literal literal = 2; + } +} + +// A statement has subject, predicate and object, and an optional context. +// The Statement message is also used for pattern queries, in which case +// non-existing fields are interpreted as wildcards. +message Statement { + Resource subject = 1; + URI predicate = 2; + Value object = 3; + Resource context = 4; +} + +// A collection of statements. +message Statements { + repeated Statement statement = 1; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/rdf_model.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/rdf_model.cc b/libraries/ostrich/backend/model/rdf_model.cc new file mode 100644 index 0000000..bd7d76d --- /dev/null +++ b/libraries/ostrich/backend/model/rdf_model.cc @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <new> + +#include "rdf_model.h" + +namespace marmotta { +namespace rdf { + +static std::string as_turtle_(const proto::URI& uri) { + return "<" + uri.uri() + ">"; +} + +static std::string as_turtle_(const proto::BNode& bnode) { + return "_:" + bnode.id(); +} + +static std::string as_turtle_(const proto::StringLiteral& literal) { + if (literal.language() == "") { + return "\"" + literal.content() + "\""; + } else { + return "\"" + literal.content() + "\"@" + literal.language(); + } +} + +static std::string as_turtle_(const proto::DatatypeLiteral& literal) { + return "\"" + literal.content() + "\"^^" + as_turtle_(literal.datatype()); +} + +static std::string as_turtle_(const proto::Resource& resource) { + if (resource.has_uri()) { + return as_turtle_(resource.uri()); + } + if (resource.has_bnode()) { + return as_turtle_(resource.bnode()); + } + return ""; +} + +static std::string as_turtle_(const proto::Value& value) { + if (value.has_resource()) { + if (value.resource().has_uri()) { + return as_turtle_(value.resource().uri()); + } + if (value.resource().has_bnode()) { + return as_turtle_(value.resource().bnode()); + } + } + if (value.has_literal()) { + if (value.literal().has_stringliteral()) { + return as_turtle_(value.literal().stringliteral()); + } + if (value.literal().has_dataliteral()) { + return as_turtle_(value.literal().dataliteral()); + } + } + return ""; +} + +std::string URI::as_turtle() const { + return as_turtle_(internal_); +} + +std::string BNode::as_turtle() const { + return as_turtle_(internal_); +} + +std::string StringLiteral::as_turtle() const { + return as_turtle_(internal_); +} + +std::string DatatypeLiteral::as_turtle() const { + return as_turtle_(internal_); +} + + + +std::string Resource::stringValue() const { + switch (type) { + case URI: + return internal_.uri().uri(); + case BNODE: + return internal_.bnode().id(); + default: + return ""; + } +} + +std::string Resource::as_turtle() const { + return as_turtle_(internal_); +} + + +Value &Value::operator=(const marmotta::rdf::URI &_uri) { + type = URI; + internal_.mutable_resource()->mutable_uri()->MergeFrom(_uri.getMessage()); + return *this; +} + + +Value &Value::operator=(const BNode &_bnode) { + type = BNODE; + internal_.mutable_resource()->mutable_bnode()->MergeFrom(_bnode.getMessage()); + return *this; +} + +Value &Value::operator=(const StringLiteral &literal) { + type = STRING_LITERAL; + internal_.mutable_literal()->mutable_stringliteral()->MergeFrom(literal.getMessage()); + return *this; +} + +Value &Value::operator=(const DatatypeLiteral &literal) { + type = DATATYPE_LITERAL; + internal_.mutable_literal()->mutable_dataliteral()->MergeFrom(literal.getMessage()); + return *this; +} + +Value &Value::operator=(marmotta::rdf::URI &&_uri) { + type = URI; + internal_.mutable_resource()->mutable_uri()->Swap(&_uri.internal_); + return *this; +} + + +Value &Value::operator=(BNode &&_bnode) { + type = BNODE; + internal_.mutable_resource()->mutable_bnode()->Swap(&_bnode.internal_); + return *this; +} + +Value &Value::operator=(StringLiteral &&literal) { + type = STRING_LITERAL; + internal_.mutable_literal()->mutable_stringliteral()->Swap(&literal.internal_); + return *this; +} + +Value &Value::operator=(DatatypeLiteral &&literal) { + type = DATATYPE_LITERAL; + internal_.mutable_literal()->mutable_dataliteral()->Swap(&literal.internal_); + return *this; +} + +std::string Value::stringValue() const { + switch (type) { + case URI: + return internal_.resource().uri().uri(); + case BNODE: + return internal_.resource().bnode().id(); + case STRING_LITERAL: + return internal_.literal().stringliteral().content(); + case DATATYPE_LITERAL: + return internal_.literal().dataliteral().content(); + default: + return ""; + } +} + +std::string Value::as_turtle() const { + return as_turtle_(internal_); +} + + +std::string Statement::as_turtle() const { + if (hasContext()) { + return as_turtle_(internal_.context()) + " { " + + as_turtle_(internal_.subject()) + " " + + as_turtle_(internal_.predicate()) + " " + + as_turtle_(internal_.object()) + ". }"; + } else { + return as_turtle_(internal_.subject()) + " " + + as_turtle_(internal_.predicate()) + " " + + as_turtle_(internal_.object()) + "."; + } +} + +Value::Value(const proto::Value& v) : internal_(v) { + if (v.has_resource()) { + if (v.resource().has_uri()) + type = URI; + else + type = BNODE; + } else if (v.has_literal()) { + if (v.literal().has_stringliteral()) + type = STRING_LITERAL; + else + type = DATATYPE_LITERAL; + } else { + type = NONE; + } +} + +Value::Value(proto::Value&& v) { + if (v.has_resource()) { + if (v.resource().has_uri()) + type = URI; + else + type = BNODE; + } else if (v.has_literal()) { + if (v.literal().has_stringliteral()) + type = STRING_LITERAL; + else + type = DATATYPE_LITERAL; + } else { + type = NONE; + } + internal_.Swap(&v); +} + +Resource::Resource(const proto::Resource& v) : internal_(v) { + if (v.has_uri()) + type = URI; + else if (v.has_bnode()) + type = BNODE; + else + type = NONE; +} + +Resource::Resource(proto::Resource&& v) { + if (v.has_uri()) + type = URI; + else if (v.has_bnode()) + type = BNODE; + else + type = NONE; + internal_.Swap(&v); +} + +Resource &Resource::operator=(const rdf::URI &uri) { + type = URI; + internal_.mutable_uri()->MergeFrom(uri.getMessage()); + return *this; +} + +Resource &Resource::operator=(const rdf::BNode &bnode) { + type = BNODE; + internal_.mutable_bnode()->MergeFrom(bnode.getMessage()); + return *this; +} + +Resource &Resource::operator=(rdf::URI &&uri) { + type = URI; + internal_.mutable_uri()->Swap(&uri.internal_); + return *this; +} + +Resource &Resource::operator=(rdf::BNode &&bnode) { + type = BNODE; + internal_.mutable_bnode()->Swap(&bnode.internal_); + return *this; +} + +URI &URI::operator=(proto::URI &&other) { + internal_.Swap(&other); + return *this; +} + +URI &URI::operator=(const URI &other) { + internal_.MergeFrom(other.internal_); + return *this; +} + +URI &URI::operator=(URI &&other) { + internal_.Swap(&other.internal_); + return *this; +} + +BNode &BNode::operator=(proto::BNode &&other) { + internal_.Swap(&other); + return *this; +} + +BNode &BNode::operator=(const BNode &other) { + internal_.MergeFrom(other.internal_); + return *this; +} + +BNode &BNode::operator=(BNode &&other) { + internal_.Swap(&other.internal_); + return *this; +} + +StringLiteral &StringLiteral::operator=(proto::StringLiteral &&other) { + internal_.Swap(&other); + return *this; +} + +StringLiteral &StringLiteral::operator=(const StringLiteral &other) { + internal_.MergeFrom(other.internal_); + return *this; +} + +StringLiteral &StringLiteral::operator=(StringLiteral &&other) { + internal_.Swap(&other.internal_); + return *this; +} + +DatatypeLiteral &DatatypeLiteral::operator=(proto::DatatypeLiteral &&other) { + internal_.Swap(&other); + return *this; +} + +DatatypeLiteral &DatatypeLiteral::operator=(const DatatypeLiteral &other) { + internal_.MergeFrom(other.internal_); + return *this; +} + +DatatypeLiteral &DatatypeLiteral::operator=(DatatypeLiteral &&other) { + internal_.Swap(&other.internal_); + return *this; +} + +Statement &Statement::operator=(const proto::Statement &other) { + internal_.MergeFrom(other); + return *this; +} + +Statement &Statement::operator=(proto::Statement &&other) { + internal_.Swap(&other); + return *this; +} + +Statement &Statement::operator=(const Statement &other) { + internal_.MergeFrom(other.internal_); + return *this; +} + +Statement &Statement::operator=(Statement &&other) { + internal_.Swap(&other.internal_); + return *this; +} +} // namespace rdf +} // namespace marmotta http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/rdf_model.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/rdf_model.h b/libraries/ostrich/backend/model/rdf_model.h new file mode 100644 index 0000000..ee5c5bb --- /dev/null +++ b/libraries/ostrich/backend/model/rdf_model.h @@ -0,0 +1,709 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MARMOTTA_RDF_MODEL_H +#define MARMOTTA_RDF_MODEL_H + +#include <string> +#include <iostream> + +#include "model/model.pb.h" + +/* + * This namespace contains the model definition for the C++ version of + * Marmotta. + * + * All objects are backed by proto messages, but offer more convenient + * high-level constructs. + * + * All objects implement copy as well as efficient move operations for + * constructors and assignment operators. Converting back and forth between + * a proto message and a model object is therefore very cheap. + */ +namespace marmotta { +namespace rdf { + +/** + * RDF namespace, consisting of a prefix and a URI. + */ +class Namespace { + public: + /* + * default constructor, creates empty namespace. + */ + Namespace() {} + + /** + * Create a new namespace from the given prefix and uri (0-terminated + * C-style strings). + */ + Namespace(const char* prefix, const char* uri) { + // Raptor sends us a nullptr for the base NS. + if (prefix != nullptr) { + internal_.set_prefix(prefix); + } + internal_.set_uri(uri); + } + + /** + * Create a new namespace from the given prefix and uri. + */ + Namespace(const std::string &prefix, const std::string &uri) { + internal_.set_prefix(prefix); + internal_.set_uri(uri); + } + + /** + * Create a new namespace from a namespace proto message. + */ + Namespace(const proto::Namespace &ns) : internal_(ns) { }; + + /** + * Create a new namespace from a namespace proto message (move constructor). + */ + Namespace(proto::Namespace &&ns) { + internal_.Swap(&ns); + }; + + /** + * Get the prefix used to identify this namespace. + */ + const std::string &getPrefix() const { + return internal_.prefix(); + } + + /** + * Set the prefix used to identify this namespace. + */ + void setPrefix(std::string &prefix) { + internal_.set_prefix(prefix); + } + + /** + * Get the URI identified by this namespace. + */ + const std::string &getUri() const { + return internal_.uri(); + } + + /** + * Set the URI identified by this namespace. + */ + void setUri(std::string &uri) { + internal_.set_uri(uri); + } + + /** + * Get a reference to the proto message wrapped by the Namespace object. + */ + const proto::Namespace& getMessage() const { + return internal_; + } + + private: + proto::Namespace internal_; +}; + +/** + * RDF URI implementation, backed by a URI proto message. + */ +class URI { + public: + /** + * Default constructor, creates an empty URI. + */ + URI() { } + + /** + * Create an URI object from the URI string passed as argument. + */ + URI(const std::string &uri) { + internal_.set_uri(uri); + } + + /** + * Create an URI object from the URI string passed as argument. + */ + URI(const char* uri) { + internal_.set_uri(uri); + } + + /** + * Create an URI object from the proto message passed as argument (copy + * constructor). + */ + URI(const proto::URI &uri) : internal_(uri) { } + + /** + * Create an URI object from the proto message passed as argument (move + * constructor, the original proto message is invalidated). + */ + URI(proto::URI &&uri) { + internal_.Swap(&uri); + } + + /** + * Copy constructor, create an URI from another URI. + */ + URI(const URI &other) : internal_(other.internal_) {}; + + /** + * Move constructor, create an URI from another URI, invalidating the + * original URI. + */ + URI(URI&& uri) { + internal_.Swap(&uri.internal_); + } + + URI & operator=(proto::URI &&other); + URI & operator=(const URI &other); + URI & operator=(URI &&other); + + /** + * Get the string representation of the URI. + */ + const std::string &getUri() const { + return internal_.uri(); + } + + /** + * Set the string representation of the URI. + */ + void setUri(std::string &uri) { + internal_.set_uri(uri); + } + + /** + * Get a canonical string representation of the URI. + */ + const std::string &stringValue() const { + return internal_.uri(); + } + + /** + * Get a Turtle representation of the URI. + */ + std::string as_turtle() const; + + /** + * Get a reference to the proto message wrapped by the URI object. + */ + const proto::URI& getMessage() const { + return internal_; + } + + private: + proto::URI internal_; + + friend class Value; + friend class Resource; + friend class Statement; +}; + +/** + * RDF Blank node implementation, backed by a BNode proto message. + */ +class BNode { + public: + /** + * Default constructor, creates empty BNode. + */ + BNode() { } + + /** + * Create a new BNode using the ID passed as argument. + */ + BNode(const std::string &id) { + internal_.set_id(id); + } + + /** + * Create a new BNode using the ID passed as argument. + */ + BNode(const char* id) { + internal_.set_id(id); + } + + /** + * Create a new BNode from the proto message passed as argument (copy + * constructor). + */ + BNode(const proto::BNode &n) : internal_(n) { } + + /** + * Create a new BNode from the proto message passed as argument (move + * constructor, original message is invalidated). + */ + BNode(proto::BNode &&n) { + internal_.Swap(&n); + }; + + /** + * Copy constructor, create a BNode from another BNode. + */ + BNode(const BNode &n) : internal_(n.internal_) {}; + + /** + * Move constructor, create a BNode from another BNode. The other BNode + * is invalidated. + */ + BNode(BNode &&n) { + internal_.Swap(&n.internal_); + }; + + BNode & operator=(proto::BNode &&other);; + BNode & operator=(const BNode &other);; + BNode & operator=(BNode &&other);; + + /** + * Return the id of this blank node. + */ + const std::string &getId() const { + return internal_.id(); + } + + /** + * Set the id of this blank node. + */ + void setId(std::string &id) { + internal_.set_id(id); + } + + /** + * Get a canonical string representation of the URI. + */ + const std::string &stringValue() const { + return internal_.id(); + } + + /** + * Get a Turtle representation of the URI. + */ + std::string as_turtle() const; + + /** + * Get a reference to the proto message wrapped by the BNode object. + */ + const proto::BNode& getMessage() const { + return internal_; + } + + private: + proto::BNode internal_; + + friend class Value; + friend class Resource; +}; + + +class StringLiteral { + public: + StringLiteral() { } + + StringLiteral(const std::string &content) { + internal_.set_content(content); + } + + StringLiteral(const std::string &content, const std::string &language) { + internal_.set_content(content); + internal_.set_language(language); + } + + StringLiteral(const proto::StringLiteral &other) : internal_(other) { }; + + StringLiteral(proto::StringLiteral &&other) { + internal_.Swap(&other); + } + + StringLiteral(const StringLiteral &other) : internal_(other.internal_) {}; + + StringLiteral(StringLiteral &&other) { + internal_.Swap(&other.internal_); + } + + StringLiteral & operator=(proto::StringLiteral &&other);; + StringLiteral & operator=(const StringLiteral &other);; + StringLiteral & operator=(StringLiteral &&other);; + + const std::string &getContent() const { + return internal_.content(); + } + + void setContent(std::string &content) { + internal_.set_content(content); + } + + const std::string &getLanguage() const { + return internal_.language(); + } + + void setLanguage(std::string &language) { + internal_.set_language(language); + } + + const std::string &stringValue() const { + return internal_.content(); + } + + const proto::StringLiteral& getMessage() const { + return internal_; + } + + std::string as_turtle() const; + + private: + proto::StringLiteral internal_; + + friend class Value; +}; + + +class DatatypeLiteral { + public: + DatatypeLiteral() { } + + DatatypeLiteral(const std::string &content, URI const &datatype) { + internal_.set_content(content); + internal_.mutable_datatype()->MergeFrom(datatype.getMessage()); + } + + DatatypeLiteral(const proto::DatatypeLiteral &other) : internal_(other) { }; + + DatatypeLiteral(proto::DatatypeLiteral &&other) { + internal_.Swap(&other); + } + + DatatypeLiteral(const DatatypeLiteral &other) : internal_(other.internal_) { }; + + DatatypeLiteral(DatatypeLiteral &&other) { + internal_.Swap(&other.internal_); + } + + DatatypeLiteral & operator=(proto::DatatypeLiteral &&other);; + DatatypeLiteral & operator=(const DatatypeLiteral &other);; + DatatypeLiteral & operator=(DatatypeLiteral &&other);; + + const std::string &getContent() const { + return internal_.content(); + } + + void setContent(std::string &content) { + internal_.set_content(content); + } + + URI getDatatype() const { + return URI(internal_.datatype()); + } + + void setDatatype(const URI &datatype) { + internal_.mutable_datatype()->MergeFrom(datatype.getMessage()); + } + + const std::string &stringValue() const { + return internal_.content(); + } + + int intValue() const { + return std::stoi(getContent()); + } + + operator int() const { + return std::stoi(getContent()); + } + + long long longValue() const { + return std::stoll(getContent()); + } + + operator long long() const { + return std::stoll(getContent()); + } + + float floatValue() const { + return std::stof(getContent()); + } + + operator float() const { + return std::stof(getContent()); + } + + double doubleValue() const { + return std::stod(getContent()); + } + + operator double() const { + return std::stod(getContent()); + } + + const proto::DatatypeLiteral& getMessage() const { + return internal_; + } + + std::string as_turtle() const; + + private: + proto::DatatypeLiteral internal_; + + friend class Value; +}; + +/** + * Value is a polymorphic, but strictly typed generic implementation for URI, + * BNode and Literal. Copy/move constructors and assignment operators allow + * using URI, BNode and Literal wherever a Value is required. + */ +class Value { + public: + enum { + URI = 1, BNODE, STRING_LITERAL, DATATYPE_LITERAL, NONE + } type; + + Value() : type(NONE) { } + + Value(const proto::Value& v); + + Value(proto::Value&& v); + + Value(const marmotta::rdf::URI &uri) : type(URI) { + internal_.mutable_resource()->mutable_uri()->MergeFrom(uri.getMessage()); + } + + Value(marmotta::rdf::URI &&uri) : type(URI) { + internal_.mutable_resource()->mutable_uri()->Swap(&uri.internal_); + } + + Value(const BNode &bnode) : type(BNODE) { + internal_.mutable_resource()->mutable_bnode()->MergeFrom(bnode.getMessage()); + } + + Value(BNode &&bnode) : type(BNODE) { + internal_.mutable_resource()->mutable_bnode()->Swap(&bnode.internal_); + } + + Value(const StringLiteral &sliteral) : type(STRING_LITERAL) { + internal_.mutable_literal()->mutable_stringliteral()->MergeFrom(sliteral.getMessage()); + }; + + Value(StringLiteral &&sliteral) : type(STRING_LITERAL) { + internal_.mutable_literal()->mutable_stringliteral()->Swap(&sliteral.internal_); + }; + + Value(const DatatypeLiteral &dliteral) : type(DATATYPE_LITERAL) { + internal_.mutable_literal()->mutable_dataliteral()->MergeFrom(dliteral.getMessage()); + }; + + Value(DatatypeLiteral &&dliteral) : type(DATATYPE_LITERAL) { + internal_.mutable_literal()->mutable_dataliteral()->Swap(&dliteral.internal_); + }; + + Value(const std::string &literal) : type(STRING_LITERAL) { + internal_.mutable_literal()->mutable_stringliteral()->set_content(literal); + }; + + Value(const char* literal) : type(STRING_LITERAL) { + internal_.mutable_literal()->mutable_stringliteral()->set_content(literal); + }; + + + Value &operator=(const rdf::URI &uri); + + Value &operator=(const rdf::BNode &bnode); + + Value &operator=(const rdf::StringLiteral &literal); + + Value &operator=(const rdf::DatatypeLiteral &literal); + + Value &operator=(rdf::URI &&uri); + + Value &operator=(rdf::BNode &&bnode); + + Value &operator=(rdf::StringLiteral &&literal); + + Value &operator=(rdf::DatatypeLiteral &&literal); + + std::string stringValue() const; + + std::string as_turtle() const; + + const proto::Value& getMessage() const { + return internal_; + } + private: + proto::Value internal_; + + friend class Statement; +}; + + +class Resource { + public: + enum { + URI, BNODE, NONE + } type; + + Resource() : type(NONE) { }; + + Resource(const proto::Resource& v); + + Resource(proto::Resource&& v); + + Resource(const std::string &uri) : type(URI) { + internal_.mutable_uri()->set_uri(uri); + }; + + Resource(const char* uri) : type(URI) { + internal_.mutable_uri()->set_uri(uri); + }; + + Resource(const rdf::URI &uri) : type(URI) { + internal_.mutable_uri()->MergeFrom(uri.getMessage()); + } + + Resource(const rdf::BNode &bnode) : type(BNODE) { + internal_.mutable_bnode()->MergeFrom(bnode.getMessage()); + } + + Resource(rdf::URI &&uri) : type(URI) { + internal_.mutable_uri()->Swap(&uri.internal_); + } + + Resource(rdf::BNode &&bnode) : type(BNODE) { + internal_.mutable_bnode()->Swap(&bnode.internal_); + } + + Resource & operator=(const rdf::URI &uri); + + Resource & operator=(const rdf::BNode &bnode); + + Resource & operator=(rdf::URI &&uri); + + Resource & operator=(rdf::BNode &&bnode); + + std::string stringValue() const; + + std::string as_turtle() const; + + const proto::Resource& getMessage() const { + return internal_; + } + private: + proto::Resource internal_; + + friend class Statement; +}; + + +class Statement { + public: + Statement() {} + + Statement(const Statement& other) : internal_(other.internal_) {} + Statement(Statement&& other) { + internal_.Swap(&other.internal_); + } + + Statement(const proto::Statement& other) : internal_(other) {} + Statement(proto::Statement&& other) { + internal_.Swap(&other); + } + + Statement & operator=(const proto::Statement &other); + Statement & operator=(proto::Statement &&other); + Statement & operator=(const Statement &other); + Statement & operator=(Statement &&other); + + + Statement(Resource const &subject, URI const &predicate, Value const &object) { + internal_.mutable_subject()->MergeFrom(subject.getMessage()); + internal_.mutable_predicate()->MergeFrom(predicate.getMessage()); + internal_.mutable_object()->MergeFrom(object.getMessage()); + } + + + Statement(Resource const &subject, URI const &predicate, Value const &object, Resource const &context) { + internal_.mutable_subject()->MergeFrom(subject.getMessage()); + internal_.mutable_predicate()->MergeFrom(predicate.getMessage()); + internal_.mutable_object()->MergeFrom(object.getMessage()); + internal_.mutable_context()->MergeFrom(context.getMessage()); + } + + Statement(Resource &&subject, URI &&predicate, Value &&object) { + internal_.mutable_subject()->Swap(&subject.internal_); + internal_.mutable_predicate()->Swap(&predicate.internal_); + internal_.mutable_object()->Swap(&object.internal_); + } + + + Statement(Resource &&subject, URI &&predicate, Value &&object, Resource &&context) { + internal_.mutable_subject()->Swap(&subject.internal_); + internal_.mutable_predicate()->Swap(&predicate.internal_); + internal_.mutable_object()->Swap(&object.internal_); + internal_.mutable_context()->Swap(&context.internal_); + } + + + Resource getSubject() const { + return Resource(internal_.subject()); + } + + void setSubject(Resource const &subject) { + internal_.mutable_subject()->MergeFrom(subject.getMessage()); + } + + URI getPredicate() const { + return URI(internal_.predicate()); + } + + void setPredicate(URI const &predicate) { + internal_.mutable_predicate()->MergeFrom(predicate.getMessage()); + } + + Value getObject() const { + return Value(internal_.object()); + } + + void setObject(Value const &object) { + internal_.mutable_object()->MergeFrom(object.getMessage()); + } + + Resource getContext() const { + return Resource(internal_.context()); + } + + void setContext(Resource const &context) { + internal_.mutable_context()->MergeFrom(context.getMessage()); + } + + bool hasContext() const { + return internal_.has_context(); + } + + std::string as_turtle() const; + + const proto::Statement& getMessage() const { + return internal_; + } + private: + proto::Statement internal_; +}; + + +} // namespace rdf +} // namespace marmotta + + +#endif //MARMOTTA_RDF_MODEL_H http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/rdf_operators.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/rdf_operators.cc b/libraries/ostrich/backend/model/rdf_operators.cc new file mode 100644 index 0000000..b9e49ad --- /dev/null +++ b/libraries/ostrich/backend/model/rdf_operators.cc @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "rdf_operators.h" + +namespace marmotta { +namespace rdf { +namespace proto { + +bool operator==(const Value &lhs, const Value &rhs) { + if (lhs.has_resource() && rhs.has_resource()) { + if (lhs.resource().has_uri() && rhs.resource().has_uri()) { + return lhs.resource().uri() == rhs.resource().uri(); + } else if (lhs.resource().has_bnode() && rhs.resource().has_bnode()) { + return lhs.resource().bnode() == rhs.resource().bnode(); + } + } else if(lhs.has_literal() && rhs.has_literal()) { + if (lhs.literal().has_stringliteral() && rhs.literal().has_stringliteral()) { + return lhs.literal().stringliteral() == rhs.literal().stringliteral(); + } else if (lhs.literal().has_dataliteral() && rhs.literal().has_dataliteral()) { + return lhs.literal().dataliteral() == rhs.literal().dataliteral(); + } + } + return false; +} + +bool operator==(const Resource &lhs, const Resource &rhs) { + if (lhs.has_uri() && rhs.has_uri()) { + return lhs.uri() == rhs.uri(); + } else if (lhs.has_bnode() && rhs.has_bnode()) { + return lhs.bnode() == rhs.bnode(); + } + return false; +} + +bool operator==(const Statement &lhs, const Statement &rhs) { + return operator==(lhs.subject(), rhs.subject()) && + operator==(lhs.predicate(), rhs.predicate()) && + operator==(lhs.object(), rhs.object()) && + operator==(lhs.context(), rhs.context()); + +} + + +} // namespace proto +} // namespace rdf +} // namespace marmotta http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/model/rdf_operators.h ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/model/rdf_operators.h b/libraries/ostrich/backend/model/rdf_operators.h new file mode 100644 index 0000000..2cf8183 --- /dev/null +++ b/libraries/ostrich/backend/model/rdf_operators.h @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MARMOTTA_RDF_OPERATORS_H +#define MARMOTTA_RDF_OPERATORS_H + +#include "model/rdf_model.h" + +namespace marmotta { +namespace rdf { +namespace proto { + +inline bool operator==(const Namespace &lhs, const Namespace &rhs) { + return lhs.uri() == rhs.uri(); +} + +inline bool operator!=(const Namespace &lhs, const Namespace &rhs) { + return lhs.uri() != rhs.uri(); +} + +inline bool operator==(const URI &lhs, const URI &rhs) { + return lhs.uri() == rhs.uri(); +} + +inline bool operator!=(const URI &lhs, const URI &rhs) { + return lhs.uri() != rhs.uri(); +} + +inline bool operator==(const BNode &lhs, const BNode &rhs) { + return lhs.id() == rhs.id(); +} + +inline bool operator!=(const BNode &lhs, const BNode &rhs) { + return lhs.id() != rhs.id(); +} + + +inline bool operator==(const StringLiteral &lhs, const StringLiteral &rhs) { + return lhs.content() == rhs.content() && lhs.language() == rhs.language(); +} + +inline bool operator!=(const StringLiteral &lhs, const StringLiteral &rhs) { + return lhs.content() != rhs.content() || lhs.language() != rhs.language(); +} + +inline bool operator==(const DatatypeLiteral &lhs, const DatatypeLiteral &rhs) { + return lhs.content() == rhs.content() && lhs.datatype().uri() == rhs.datatype().uri(); +} + +inline bool operator!=(const DatatypeLiteral &lhs, const DatatypeLiteral &rhs) { + return lhs.content() != rhs.content() || lhs.datatype().uri() != rhs.datatype().uri(); +} + +bool operator==(const Value &lhs, const Value &rhs); + +inline bool operator!=(const Value &lhs, const Value &rhs) { + return !operator==(lhs,rhs); +}; + + +bool operator==(const Resource &lhs, const Resource &rhs); + +inline bool operator!=(const Resource &lhs, const Resource &rhs) { + return !operator==(lhs,rhs); +}; + +bool operator==(const Statement &lhs, const Statement &rhs); + +inline bool operator!=(const Statement &lhs, const Statement &rhs) { + return !operator==(lhs,rhs); +}; + + +} // namespace proto + + +inline bool operator==(const Namespace &lhs, const Namespace &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const Namespace &lhs, const Namespace &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const URI &lhs, const URI &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const URI &lhs, const URI &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const BNode &lhs, const BNode &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const BNode &lhs, const BNode &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const StringLiteral &lhs, const StringLiteral &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const StringLiteral &lhs, const StringLiteral &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const DatatypeLiteral &lhs, const DatatypeLiteral &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const DatatypeLiteral &lhs, const DatatypeLiteral &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const Value &lhs, const Value &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const Value &lhs, const Value &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const Resource &lhs, const Resource &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const Resource &lhs, const Resource &rhs) { + return !operator==(lhs,rhs); +} + +inline bool operator==(const Statement &lhs, const Statement &rhs) { + return lhs.getMessage() == rhs.getMessage(); +} + +inline bool operator!=(const Statement &lhs, const Statement &rhs) { + return !operator==(lhs,rhs); +} + +} // namespace rdf +} // namespace marmotta + +namespace std { + +// Define std::hash specializations for our proto messages. Note that this generic +// computation serializes the message and is therefore expensive. Consider using +// specialised implementations instead. +template<> +struct hash<google::protobuf::Message> { + std::size_t operator()(const google::protobuf::Message &k) const { + std::string content; + k.SerializeToString(&content); + return std::hash<string>()(content); + } +}; + +// Hash implementation for URIs. Uses a faster implementation than the generic +// proto message version. +template<> +struct hash<marmotta::rdf::proto::URI> { + std::size_t operator()(const marmotta::rdf::proto::URI &k) const { + return std::hash<std::string>()(k.uri()); + } +}; + +// Hash implementation for BNodes. Uses a faster implementation than the generic +// proto message version. +template<> +struct hash<marmotta::rdf::proto::BNode> { + std::size_t operator()(const marmotta::rdf::proto::BNode &k) const { + return std::hash<std::string>()(k.id()); + } +}; + +// Hash implementation for Resources. Uses a faster implementation than the generic +// proto message version. +template<> +struct hash<marmotta::rdf::proto::Resource> { + std::size_t operator()(const marmotta::rdf::proto::Resource &k) const { + if (k.has_uri()) { + return std::hash<marmotta::rdf::proto::URI>()(k.uri()); + } else if (k.has_bnode()) { + return std::hash<marmotta::rdf::proto::BNode>()(k.bnode()); + } + return std::hash<google::protobuf::Message>()(k); + } +}; + +template<> +struct hash<marmotta::rdf::proto::Value> { + std::size_t operator()(const marmotta::rdf::proto::Value &k) const { + return std::hash<google::protobuf::Message>()(k); + } +}; + +template<> +struct hash<marmotta::rdf::proto::StringLiteral> { + std::size_t operator()(const marmotta::rdf::proto::StringLiteral &k) const { + return std::hash<google::protobuf::Message>()(k); + } +}; + +template<> +struct hash<marmotta::rdf::proto::DatatypeLiteral> { + std::size_t operator()(const marmotta::rdf::proto::DatatypeLiteral &k) const { + return std::hash<google::protobuf::Message>()(k); + } +}; + +template<> +struct hash<marmotta::rdf::proto::Statement> { + std::size_t operator()(const marmotta::rdf::proto::Statement &k) const { + return std::hash<google::protobuf::Message>()(k); + } +}; + +template<> +struct hash<marmotta::rdf::proto::Namespace> { + std::size_t operator()(const marmotta::rdf::proto::Namespace &k) const { + return std::hash<google::protobuf::Message>()(k); + } +}; +} // namespace std + +#endif //MARMOTTA_RDF_OPERATORS_H http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/parser/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/parser/CMakeLists.txt b/libraries/ostrich/backend/parser/CMakeLists.txt new file mode 100644 index 0000000..3ed5634 --- /dev/null +++ b/libraries/ostrich/backend/parser/CMakeLists.txt @@ -0,0 +1,4 @@ +include_directories(.. ${CMAKE_CURRENT_BINARY_DIR}/..) + +add_library(marmotta_parser rdf_parser.h rdf_parser.cc) +target_link_libraries(marmotta_parser marmotta_model ${CMAKE_THREAD_LIBS_INIT} ${RAPTOR_LIBRARY}) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/marmotta/blob/0ff22a0c/libraries/ostrich/backend/parser/rdf_parser.cc ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/parser/rdf_parser.cc b/libraries/ostrich/backend/parser/rdf_parser.cc new file mode 100644 index 0000000..9a1fea3 --- /dev/null +++ b/libraries/ostrich/backend/parser/rdf_parser.cc @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <raptor2/raptor2.h> +#include "rdf_parser.h" + +namespace marmotta { +namespace parser { +Parser::Parser(const rdf::URI& baseUri, Format format) + : stmt_handler([](const rdf::Statement& stmt) { }) + , ns_handler([](const rdf::Namespace& ns) { }) +{ + world = raptor_new_world(); + base = raptor_new_uri(world, (unsigned char const *) baseUri.getUri().c_str()); + + switch (format) { + case RDFXML: + parser = raptor_new_parser(world, "rdfxml"); + break; + case TURTLE: + parser = raptor_new_parser(world, "turtle"); + break; + case NTRIPLES: + parser = raptor_new_parser(world, "ntriples"); + break; + case RDFA: + parser = raptor_new_parser(world, "rdfa"); + break; + case RDFJSON: + parser = raptor_new_parser(world, "json"); + break; + case GUESS: + parser = raptor_new_parser(world, "guess"); + break; + } + + raptor_parser_set_statement_handler(parser, this, raptor_stmt_handler); + raptor_parser_set_namespace_handler(parser, this, raptor_ns_handler); +} + +Parser::~Parser() { + raptor_free_parser(parser); + raptor_free_uri(base); + raptor_free_world(world); +} + + +void Parser::raptor_stmt_handler(void *user_data, raptor_statement *statement) { + Parser* p = static_cast<Parser*>(user_data); + + rdf::Resource subject; rdf::URI predicate; rdf::Value object; rdf::Resource context; + switch (statement->subject->type) { + case RAPTOR_TERM_TYPE_URI: + subject = rdf::URI((const char*)raptor_uri_as_string(statement->subject->value.uri)); + break; + case RAPTOR_TERM_TYPE_BLANK: + subject = rdf::BNode(std::string((const char*)statement->subject->value.blank.string, statement->subject->value.blank.string_len)); + break; + default: + raptor_parser_parse_abort(p->parser); + throw ParseError("invalid subject term type"); + } + + switch (statement->predicate->type) { + case RAPTOR_TERM_TYPE_URI: + predicate = rdf::URI((const char*)raptor_uri_as_string(statement->predicate->value.uri)); + break; + default: + raptor_parser_parse_abort(p->parser); + throw ParseError("invalid predicate term type"); + } + + switch (statement->object->type) { + case RAPTOR_TERM_TYPE_URI: + object = rdf::URI((const char*)raptor_uri_as_string(statement->object->value.uri)); + break; + case RAPTOR_TERM_TYPE_BLANK: + object = rdf::BNode(std::string((const char*)statement->object->value.blank.string, statement->object->value.blank.string_len)); + break; + case RAPTOR_TERM_TYPE_LITERAL: + if(statement->object->value.literal.language != NULL) { + object = rdf::StringLiteral( + std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len), + std::string((const char*)statement->object->value.literal.language, statement->object->value.literal.language_len) + ); + } else if(statement->object->value.literal.datatype != NULL) { + object = rdf::DatatypeLiteral( + std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len), + rdf::URI((const char*)raptor_uri_as_string(statement->object->value.literal.datatype)) + ); + } else { + object = rdf::StringLiteral( + std::string((const char*)statement->object->value.literal.string, statement->object->value.literal.string_len) + ); + } + break; + default: + raptor_parser_parse_abort(p->parser); + throw ParseError("invalid object term type"); + } + + if (statement->graph != NULL) { + switch (statement->graph->type) { + case RAPTOR_TERM_TYPE_URI: + context = rdf::URI((const char*)raptor_uri_as_string(statement->graph->value.uri)); + break; + case RAPTOR_TERM_TYPE_BLANK: + context = rdf::BNode(std::string((const char*)statement->graph->value.blank.string, statement->graph->value.blank.string_len)); + break; + default: + raptor_parser_parse_abort(p->parser); + throw ParseError("invalid graph term type"); + } + } else { + context = rdf::URI(); + } + + p->stmt_handler(rdf::Statement(subject, predicate, object, context)); +} + + +void Parser::raptor_ns_handler(void *user_data, raptor_namespace *nspace) { + Parser* p = static_cast<Parser*>(user_data); + p->ns_handler(rdf::Namespace( + (const char*)raptor_namespace_get_prefix(nspace), + (const char*)raptor_uri_as_string(raptor_namespace_get_uri(nspace)))); +} + +void Parser::parse(std::istream &in) { + if(in) { + raptor_parser_parse_start(parser, base); + + char buffer[8192]; + while (in.read(buffer, 8192)) { + raptor_parser_parse_chunk(parser, (unsigned char const *) buffer, in.gcount(), 0); + } + raptor_parser_parse_chunk(parser, (unsigned char const *) buffer, in.gcount(), 1); + } +} + +Format FormatFromString(const std::string &name) { + if (name == "rdfxml" || name == "rdf/xml" || name == "xml") { + return RDFXML; + } + if (name == "n3" || name == "ntriples" || name == "text/n3") { + return NTRIPLES; + } + if (name == "turtle" || name == "text/turtle") { + return TURTLE; + } + if (name == "json" || name == "application/json" || name == "application/rdf+json") { + return RDFJSON; + } + if (name == "auto" || name == "guess") { + return GUESS; + } + return RDFXML; +} + +} +}
