Return-Path: X-Original-To: apmail-marmotta-commits-archive@minotaur.apache.org Delivered-To: apmail-marmotta-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0A21418A97 for ; Sun, 20 Dec 2015 11:39:18 +0000 (UTC) Received: (qmail 45892 invoked by uid 500); 20 Dec 2015 11:39:17 -0000 Delivered-To: apmail-marmotta-commits-archive@marmotta.apache.org Received: (qmail 45855 invoked by uid 500); 20 Dec 2015 11:39:17 -0000 Mailing-List: contact commits-help@marmotta.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@marmotta.apache.org Delivered-To: mailing list commits@marmotta.apache.org Received: (qmail 45846 invoked by uid 99); 20 Dec 2015 11:39:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 20 Dec 2015 11:39:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 684D2DFF70; Sun, 20 Dec 2015 11:39:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sschaffert@apache.org To: commits@marmotta.apache.org Message-Id: <5d445004e7e64477b29c25dd51f86a72@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: marmotta git commit: [MARMOTTA-621] implement a loader backend for Ostrich, experiment with DBPedia gives arounf 45k triples/sec Date: Sun, 20 Dec 2015 11:39:17 +0000 (UTC) Repository: marmotta Updated Branches: refs/heads/develop 4eb074dcb -> 8f60cf64d [MARMOTTA-621] implement a loader backend for Ostrich, experiment with DBPedia gives arounf 45k triples/sec Project: http://git-wip-us.apache.org/repos/asf/marmotta/repo Commit: http://git-wip-us.apache.org/repos/asf/marmotta/commit/8f60cf64 Tree: http://git-wip-us.apache.org/repos/asf/marmotta/tree/8f60cf64 Diff: http://git-wip-us.apache.org/repos/asf/marmotta/diff/8f60cf64 Branch: refs/heads/develop Commit: 8f60cf64dc8b8e99d89dbf4e54fa81af3635606f Parents: 4eb074d Author: Sebastian Schaffert Authored: Sun Dec 20 12:40:52 2015 +0100 Committer: Sebastian Schaffert Committed: Sun Dec 20 12:40:52 2015 +0100 ---------------------------------------------------------------------- libraries/ostrich/backend/CMakeLists.txt | 2 +- .../ostrich/sail/OstrichSailConnection.java | 14 +- loader/marmotta-loader-ostrich/pom.xml | 83 ++++++++++ .../loader/ostrich/OstrichLoaderBackend.java | 100 ++++++++++++ .../loader/ostrich/OstrichLoaderHandler.java | 156 +++++++++++++++++++ ...org.apache.marmotta.loader.api.LoaderBackend | 18 +++ .../src/main/resources/logback.xml | 32 ++++ loader/pom.xml | 10 ++ 8 files changed, 407 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/marmotta/blob/8f60cf64/libraries/ostrich/backend/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/libraries/ostrich/backend/CMakeLists.txt b/libraries/ostrich/backend/CMakeLists.txt index 5a8f110..61156a5 100644 --- a/libraries/ostrich/backend/CMakeLists.txt +++ b/libraries/ostrich/backend/CMakeLists.txt @@ -17,7 +17,7 @@ find_package (GLog REQUIRED) find_package (Boost 1.54.0 COMPONENTS iostreams filesystem system) find_package (Tcmalloc) -#add_definitions(-DNDEBUG) +add_definitions(-DNDEBUG) if (Boost_IOSTREAMS_FOUND) message(STATUS "Enabling gzip/bzip2 support (Boost iostreams found)") http://git-wip-us.apache.org/repos/asf/marmotta/blob/8f60cf64/libraries/ostrich/client/src/main/java/org/apache/marmotta/ostrich/sail/OstrichSailConnection.java ---------------------------------------------------------------------- diff --git a/libraries/ostrich/client/src/main/java/org/apache/marmotta/ostrich/sail/OstrichSailConnection.java b/libraries/ostrich/client/src/main/java/org/apache/marmotta/ostrich/sail/OstrichSailConnection.java index 93e21ac..e85fdd2 100644 --- a/libraries/ostrich/client/src/main/java/org/apache/marmotta/ostrich/sail/OstrichSailConnection.java +++ b/libraries/ostrich/client/src/main/java/org/apache/marmotta/ostrich/sail/OstrichSailConnection.java @@ -107,7 +107,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void addStatementInternal(Resource subj, URI pred, Value obj, Resource... contexts) throws SailException { - log.info("Adding statements."); + log.debug("Adding statements."); ensureTransaction(); if (contexts.length > 0) { @@ -319,7 +319,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void removeStatementsInternal(Resource subj, URI pred, Value obj, Resource... contexts) throws SailException { - log.info("Removing statements."); + log.debug("Removing statements."); commitForQuery(); ensureTransaction(); @@ -338,7 +338,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void clearInternal(Resource... contexts) throws SailException { - log.info("Clearing statements."); + log.debug("Clearing statements."); commitForQuery(); ensureTransaction(); @@ -357,7 +357,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected CloseableIteration getNamespacesInternal() throws SailException { - log.info("Getting namespaces."); + log.debug("Getting namespaces."); commitForQuery(); Empty pattern = Empty.getDefaultInstance(); @@ -382,7 +382,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void setNamespaceInternal(String prefix, String name) throws SailException { - log.info("Setting namespace {} = {}.", prefix, name); + log.debug("Setting namespace {} = {}.", prefix, name); ensureTransaction(); ProtoNamespace ns = new ProtoNamespace(prefix, name); @@ -393,7 +393,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void removeNamespaceInternal(String prefix) throws SailException { - log.info("Removing namespace {}.", prefix); + log.debug("Removing namespace {}.", prefix); commitForQuery(); ensureTransaction(); @@ -404,7 +404,7 @@ public class OstrichSailConnection extends NotifyingSailConnectionBase { @Override protected void clearNamespacesInternal() throws SailException { - log.info("Clearing namespaces."); + log.debug("Clearing namespaces."); commitForQuery(); ensureTransaction(); http://git-wip-us.apache.org/repos/asf/marmotta/blob/8f60cf64/loader/marmotta-loader-ostrich/pom.xml ---------------------------------------------------------------------- diff --git a/loader/marmotta-loader-ostrich/pom.xml b/loader/marmotta-loader-ostrich/pom.xml new file mode 100644 index 0000000..5816b68 --- /dev/null +++ b/loader/marmotta-loader-ostrich/pom.xml @@ -0,0 +1,83 @@ + + + + 4.0.0 + + + org.apache.marmotta + marmotta-parent + 3.4.0-SNAPSHOT + ../../parent + + + marmotta-loader-ostrich + Loader: Ostrich Backend + + + Apache Marmotta bulk loader backend for loading large datasets into a Ostrich/LevelDB backend. + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.2 + + + package + + shade + + + false + + + + org.apache.marmotta.loader.core.MarmottaLoader + + + + false + + + + + + + + + + + + org.apache.marmotta + marmotta-loader-core + ${project.version} + + + org.apache.marmotta + ostrich-model + ${project.version} + + + org.apache.marmotta + ostrich-client + ${project.version} + + + http://git-wip-us.apache.org/repos/asf/marmotta/blob/8f60cf64/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java ---------------------------------------------------------------------- diff --git a/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java b/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java new file mode 100644 index 0000000..bd6e49e --- /dev/null +++ b/loader/marmotta-loader-ostrich/src/main/java/org/apache/marmotta/loader/ostrich/OstrichLoaderBackend.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.marmotta.loader.ostrich; + +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.configuration.Configuration; +import org.apache.marmotta.loader.api.LoaderBackend; +import org.apache.marmotta.loader.api.LoaderHandler; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * Ostrich loader backend. Provides configuration for the OstrichLoaderHandler. + * + * @author Sebastian Schaffert (sschaffert@apache.org) + */ +public class OstrichLoaderBackend implements LoaderBackend { + + /** + * Create the RDFHandler to be used for bulk-loading, optionally using the configuration passed as argument. + * + * @param configuration + * @return a newly created RDFHandler instance + */ + @Override + public LoaderHandler createLoader(Configuration configuration) { + return new OstrichLoaderHandler( + configuration.getString("backend.ostrich.host", "localhost"), + configuration.getInt("backend.ostrich.port", 10000), + configuration.getLong("backend.ostrich.batchsize", 500000)); + } + + /** + * Return a unique identifier for the loader; used for identifying the loader to choose on the command line + * in case more than one loader implementation is available. + *

+ * Should match with the regular expression [a-z][a-z0-9]* + * + * @return + */ + @Override + public String getIdentifier() { + return "ostrich"; + } + + /** + * Return any additional options that this backend offers (e.g. for connecting to a database etc). + * If there are no additional options, return an empty collection. + * + * @return + */ + @Override + public Collection