Return-Path: X-Original-To: apmail-marmotta-commits-archive@minotaur.apache.org Delivered-To: apmail-marmotta-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D16D510ADE for ; Mon, 30 Sep 2013 23:49:44 +0000 (UTC) Received: (qmail 26075 invoked by uid 500); 30 Sep 2013 23:49:44 -0000 Delivered-To: apmail-marmotta-commits-archive@marmotta.apache.org Received: (qmail 26046 invoked by uid 500); 30 Sep 2013 23:49:44 -0000 Mailing-List: contact commits-help@marmotta.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@marmotta.incubator.apache.org Delivered-To: mailing list commits@marmotta.incubator.apache.org Received: (qmail 26039 invoked by uid 99); 30 Sep 2013 23:49:44 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Sep 2013 23:49:44 +0000 X-ASF-Spam-Status: No, hits=-2000.7 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Mon, 30 Sep 2013 23:48:19 +0000 Received: (qmail 24755 invoked by uid 99); 30 Sep 2013 23:47:54 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Sep 2013 23:47:54 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 856FF833C91; Mon, 30 Sep 2013 23:47:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ansell@apache.org To: commits@marmotta.incubator.apache.org Date: Mon, 30 Sep 2013 23:48:08 -0000 Message-Id: <8885cb06239346b1902f86d0a62d3e2f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/51] [partial] Revert "Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-marmotta" X-Virus-Checked: Checked by ClamAV on apache.org http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java index 7ec1109..27927f3 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/KiWiPersistence.java @@ -18,12 +18,15 @@ package org.apache.marmotta.kiwi.persistence; import org.apache.marmotta.kiwi.caching.KiWiCacheManager; -import org.apache.marmotta.kiwi.config.KiWiConfiguration; -import org.apache.marmotta.kiwi.generator.*; +import org.apache.marmotta.kiwi.model.rdf.KiWiNode; +import org.apache.marmotta.kiwi.model.rdf.KiWiResource; +import org.apache.marmotta.kiwi.model.rdf.KiWiUriResource; import org.apache.marmotta.kiwi.persistence.util.ScriptRunner; -import org.apache.marmotta.kiwi.sail.KiWiValueFactory; import org.apache.tomcat.jdbc.pool.DataSource; import org.apache.tomcat.jdbc.pool.PoolProperties; +import org.openrdf.model.Statement; +import org.openrdf.repository.RepositoryException; +import org.openrdf.repository.RepositoryResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +34,9 @@ import java.io.IOException; import java.io.StringReader; import java.sql.Connection; import java.sql.SQLException; +import java.util.Collections; import java.util.Set; +import java.util.WeakHashMap; /** * Add file description here! @@ -46,54 +51,34 @@ public class KiWiPersistence { private static int KIWI_ID = 0; /** - * The connection pool for managing JDBC connections + * A unique name for identifying this instance of KiWiPersistence. Can be used in case there are several + * instances running in the same environment. */ - private DataSource connectionPool; - - private PoolProperties poolConfig; - - private KiWiCacheManager cacheManager; + private String name; - private KiWiGarbageCollector garbageCollector; /** - * The KiWi configuration for this persistence. + * The connection pool for managing JDBC connections */ - private KiWiConfiguration configuration; + private DataSource connectionPool; /** - * A reference to the value factory used to access this store. Used for notifications when to flush batches. + * The SQL dialect to use */ - private KiWiValueFactory valueFactory; - - - private IDGenerator idGenerator; - + private KiWiDialect dialect; - /** - * This lock allows setting the backend into maintenance mode (by locking the write lock), which essentially - * grants an exclusive access to the database. This is currently used by the garbage collector, but can also - * be used in other situations- - */ - private boolean maintenance; + private PoolProperties poolConfig; - private boolean droppedDatabase = false; + private KiWiCacheManager cacheManager; - private boolean initialized = false; + private KiWiGarbageCollector garbageCollector; - @Deprecated public KiWiPersistence(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect dialect) { - this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect)); - } + this.name = name; + this.dialect = dialect; - public KiWiPersistence(KiWiConfiguration configuration) { - this.configuration = configuration; - this.maintenance = false; - } - - public void initialise() { // init JDBC connection pool - initConnectionPool(); + initConnectionPool(jdbcUrl, db_user, db_password); // init EHCache caches initCachePool(); @@ -107,14 +92,10 @@ public class KiWiPersistence { } - //garbageCollector.start(); - - initialized = true; } - public KiWiDialect getDialect() { - return configuration.getDialect(); + return dialect; } public KiWiCacheManager getCacheManager() { @@ -123,39 +104,30 @@ public class KiWiPersistence { private void initCachePool() { - cacheManager = new KiWiCacheManager(configuration.getName()); + cacheManager = new KiWiCacheManager(name); } - private void initConnectionPool() { + private void initConnectionPool(String jdbcUrl, String db_user, String db_password) { poolConfig = new PoolProperties(); poolConfig.setName("kiwi-" + (++KIWI_ID)); - poolConfig.setUrl(configuration.getJdbcUrl()); - poolConfig.setDriverClassName(configuration.getDialect().getDriverClass()); - poolConfig.setUsername(configuration.getDbUser()); - poolConfig.setPassword(configuration.getDbPassword()); + poolConfig.setUrl(jdbcUrl); + poolConfig.setDriverClassName(dialect.getDriverClass()); + poolConfig.setUsername(db_user); + poolConfig.setPassword(db_password); poolConfig.setDefaultTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); poolConfig.setCommitOnReturn(true); - poolConfig.setValidationQuery(configuration.getDialect().getValidationQuery()); - poolConfig.setLogValidationErrors(true); /* poolConfig.setLogAbandoned(true); poolConfig.setRemoveAbandoned(true); */ // interceptors - if(configuration.isQueryLoggingEnabled()) { - poolConfig.setJdbcInterceptors( - "org.apache.tomcat.jdbc.pool.interceptor.ConnectionState;" + - "org.apache.tomcat.jdbc.pool.interceptor.StatementFinalizer;" + - "org.apache.tomcat.jdbc.pool.interceptor.SlowQueryReport" - ); - } else { - poolConfig.setJdbcInterceptors( - "org.apache.tomcat.jdbc.pool.interceptor.ConnectionState;" + - "org.apache.tomcat.jdbc.pool.interceptor.StatementFinalizer" - ); - } + poolConfig.setJdbcInterceptors( + "org.apache.tomcat.jdbc.pool.interceptor.ConnectionState;" + + "org.apache.tomcat.jdbc.pool.interceptor.StatementFinalizer;" + + "org.apache.tomcat.jdbc.pool.interceptor.SlowQueryReport" + ); if(log.isDebugEnabled()) { poolConfig.setSuspectTimeout(30); @@ -179,39 +151,9 @@ public class KiWiPersistence { } - /** - * Initialise in-memory sequences if the feature is enabled. - */ - public void initSequences(String scriptName) { - switch (configuration.getIdGeneratorType()) { - case DATABASE_SEQUENCE: - idGenerator = new DatabaseSequenceIDGenerator(); - break; - case MEMORY_SEQUENCE: - idGenerator = new MemorySequenceIDGenerator(); - break; - case UUID_TIME: - idGenerator = new UUIDTimeIDGenerator(); - break; - case UUID_RANDOM: - idGenerator = new UUIDRandomIDGenerator(); - break; - case SNOWFLAKE: - idGenerator = new SnowflakeIDGenerator(); - break; - default: - idGenerator = new DatabaseSequenceIDGenerator(); - } - idGenerator.init(this,scriptName); - } - public void logPoolInfo() throws SQLException { - if(connectionPool != null) { - log.debug("num_busy_connections: {}", connectionPool.getNumActive()); - log.debug("num_idle_connections: {}", connectionPool.getNumIdle()); - } else { - log.debug("connection pool not initialized"); - } + log.debug("num_busy_connections: {}", connectionPool.getNumActive()); + log.debug("num_idle_connections: {}", connectionPool.getNumIdle()); } @@ -248,14 +190,14 @@ public class KiWiPersistence { log.info("creating new KiWi database ..."); ScriptRunner runner = new ScriptRunner(connection.getJDBCConnection(), false, false); - runner.runScript(new StringReader(configuration.getDialect().getCreateScript(scriptName))); + runner.runScript(new StringReader(dialect.getCreateScript(scriptName))); } else { int version = connection.getDatabaseVersion(); - String updateScript = configuration.getDialect().getMigrationScript(version,scriptName); + String updateScript = dialect.getMigrationScript(version,scriptName); if(updateScript != null && updateScript.length() > 0) { - log.info("upgrading existing KiWi database from version {} to version {}", version, configuration.getDialect().getVersion()); + log.info("upgrading existing KiWi database from version {} to version {}", version, dialect.getVersion()); ScriptRunner runner = new ScriptRunner(connection.getJDBCConnection(), false, false); runner.runScript(new StringReader(updateScript)); @@ -264,7 +206,7 @@ public class KiWiPersistence { log.info("connecting to existing KiWi database (version: {})",version); } } - connection.getJDBCConnection().commit(); + connection.commit(); } catch (SQLException ex) { log.error("SQL exception while initialising database, rolling back"); connection.rollback(); @@ -275,9 +217,6 @@ public class KiWiPersistence { } finally { connection.close(); } - - // init the in-memory sequences - initSequences(scriptName); } /** @@ -319,7 +258,7 @@ public class KiWiPersistence { } ScriptRunner runner = new ScriptRunner(connection.getJDBCConnection(), false, false); - runner.runScript(new StringReader(configuration.getDialect().getDropScript(scriptName))); + runner.runScript(new StringReader(dialect.getDropScript(scriptName))); if(log.isDebugEnabled()) { @@ -329,7 +268,7 @@ public class KiWiPersistence { log.debug("- found table: {}",table); } } - connection.getJDBCConnection().commit(); + connection.commit(); } catch (SQLException ex) { log.error("SQL exception while dropping database, rolling back"); connection.rollback(); @@ -343,8 +282,6 @@ public class KiWiPersistence { } catch(SQLException ex) { log.error("SQL exception while acquiring database connection"); } - - droppedDatabase = true; } /** @@ -354,20 +291,7 @@ public class KiWiPersistence { * @throws SQLException in case a new connection could not be established */ public KiWiConnection getConnection() throws SQLException { - if(!initialized) { - throw new SQLException("persistence backend not initialized; call initialise before acquiring a connection"); - } - - if(connectionPool != null) { - KiWiConnection con = new KiWiConnection(this,configuration.getDialect(),cacheManager); - if(getDialect().isBatchSupported()) { - con.setBatchCommit(configuration.isBatchCommit()); - con.setBatchSize(configuration.getBatchSize()); - } - return con; - } else { - throw new SQLException("connection pool is closed, database connections not available"); - } + return new KiWiConnection(this,dialect,cacheManager); } /** @@ -376,60 +300,18 @@ public class KiWiPersistence { * @throws SQLException */ public Connection getJDBCConnection() throws SQLException { - return getJDBCConnection(false); - } + Connection conn = connectionPool.getConnection(); + conn.setAutoCommit(false); + //conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); - /** - * Return a raw JDBC connection from the connection pool, which already has the auto-commit disabled. - * @return - * @throws SQLException - */ - public Connection getJDBCConnection(boolean maintenance) throws SQLException { - synchronized (this) { - if(this.maintenance) { - try { - this.wait(); - } catch (InterruptedException e) { } - } - if(maintenance) { - this.maintenance = true; - } - } - - if(initialized && connectionPool != null) { - Connection conn = connectionPool.getConnection(); - conn.setAutoCommit(false); + //managedConnections.add(conn); - return conn; - } else { - throw new SQLException("connection pool is closed, database connections not available"); - } + return conn; } - /** - * Release the JDBC connection passed as argument. This method will close the connection and release - * any locks that might be held by the caller. - * @param con - * @throws SQLException - */ - public void releaseJDBCConnection(Connection con) throws SQLException { - try { - con.close(); - } finally { - synchronized (this) { - if(this.maintenance) { - this.maintenance = false; - this.notifyAll(); - } - } - } - } - private void forceCloseConnections() { - if(connectionPool != null) { - connectionPool.close(true); - } + connectionPool.close(true); connectionPool = new DataSource(poolConfig); } @@ -458,16 +340,63 @@ public class KiWiPersistence { garbageCollector.addTripleTableDependency(tableName, columnName); } + /** + * Return a Sesame RepositoryResult of statements according to the query pattern given in the arguments. Each of + * the parameters subject, predicate, object and context may be null, indicating a wildcard query. If the boolean + * parameter "inferred" is set to true, the result will also include inferred triples, if it is set to false only + * base triples. + *

+ * The RepositoryResult holds a direct connection to the database and needs to be closed properly, or otherwise + * the system might run out of resources. The returned RepositoryResult will try its best to clean up when the + * iteration has completed or the garbage collector calls the finalize() method, but this can take longer than + * necessary. + *

+ * This method will create a new database connection for running the query which is only released when the + * result is closed. + * + * + * @param subject the subject to query for, or null for a wildcard query + * @param predicate the predicate to query for, or null for a wildcard query + * @param object the object to query for, or null for a wildcard query + * @param context the context to query for, or null for a wildcard query + * @param inferred if true, the result will also contain triples inferred by the reasoner, if false not + * @return a new RepositoryResult with a direct connection to the database; the result should be properly closed + * by the caller + */ + public RepositoryResult listTriples(KiWiResource subject, KiWiUriResource predicate, KiWiNode object, KiWiResource context, boolean inferred) throws SQLException { + final KiWiConnection conn = getConnection(); + + return new RepositoryResult(conn.listTriples(subject,predicate,object,context,inferred)) { + @Override + protected void handleClose() throws RepositoryException { + super.handleClose(); + try { + if(!conn.isClosed()) { + conn.commit(); + conn.close(); + } + } catch (SQLException ex) { + throw new RepositoryException("SQL error when closing database connection",ex); + } + } + + @Override + protected void finalize() throws Throwable { + handleClose(); + super.finalize(); + } + }; + } - public void shutdown() { - initialized = false; - idGenerator.shutdown(this); + public void initialise() { + garbageCollector.start(); + } + + public void shutdown() { garbageCollector.shutdown(); cacheManager.shutdown(); connectionPool.close(); - - connectionPool = null; } /** @@ -478,28 +407,4 @@ public class KiWiPersistence { } - public void setValueFactory(KiWiValueFactory valueFactory) { - this.valueFactory = valueFactory; - } - - public KiWiValueFactory getValueFactory() { - return valueFactory; - } - - public KiWiConfiguration getConfiguration() { - return configuration; - } - - - public void garbageCollect() throws SQLException { - this.garbageCollector.garbageCollect(); - } - - public boolean checkConsistency() throws SQLException { - return garbageCollector.checkConsistency(); - } - - public IDGenerator getIdGenerator() { - return idGenerator; - } } http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/h2/H2Dialect.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/h2/H2Dialect.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/h2/H2Dialect.java index 8bbb1a6..dc57948 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/h2/H2Dialect.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/persistence/h2/H2Dialect.java @@ -44,47 +44,4 @@ public class H2Dialect extends KiWiDialect { public String getDriverClass() { return "org.h2.Driver"; } - - @Override - public boolean isBatchSupported() { - return false; - } - - @Override - public String getRegexp(String text, String pattern) { - return text + " REGEXP " + pattern; - } - - @Override - public String getILike(String text, String pattern) { - return "lower("+text+") LIKE lower("+pattern+")"; - } - - - - @Override - public String getConcat(String... args) { - StringBuilder buf = new StringBuilder(); - buf.append("CONCAT("); - for(int i=0; i * Author: Sebastian Schaffert */ -public class KiWiSailConnection extends NotifyingSailConnectionBase implements InferencerConnection, ResourceConnection { +public class KiWiSailConnection extends NotifyingSailConnectionBase implements InferencerConnection { private static final Logger log = LoggerFactory.getLogger(KiWiSailConnection.class); @@ -82,6 +89,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I private boolean triplesAdded, triplesRemoved; + private HashSet deletedStatementsLog; public KiWiSailConnection(KiWiStore sailBase) throws SailException { super(sailBase); @@ -141,14 +149,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I } } if(contextSet.size() == 0) { - if(defaultContext != null) { - contextSet.add(valueFactory.createURI(defaultContext)); - } else { - contextSet.add(null); - } - } - if(inferred && inferredContext != null) { - contextSet.add(valueFactory.createURI(inferredContext)); + contextSet.add(valueFactory.createURI(defaultContext)); } KiWiResource ksubj = valueFactory.convert(subj); @@ -163,7 +164,16 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I KiWiTriple triple = (KiWiTriple)valueFactory.createStatement(ksubj,kpred,kobj,kcontext, databaseConnection); triple.setInferred(inferred); - if(databaseConnection.storeTriple(triple)) { + if(triple.getId() == null) { + databaseConnection.storeTriple(triple); + triplesAdded = true; + notifyStatementAdded(triple); + } else if(deletedStatementsLog.contains(triple.getId())) { + // this is a hack for a concurrency problem that may occur in case the triple is removed in the + // transaction and then added again; in these cases the createStatement method might return + // an expired state of the triple because it uses its own database connection + + databaseConnection.undeleteTriple(triple); triplesAdded = true; notifyStatementAdded(triple); } @@ -228,7 +238,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I new FilterOptimizer().optimize(tupleExpr, dataset, bindings); new OrderLimitOptimizer().optimize(tupleExpr, dataset, bindings); - return strategy.evaluate(tupleExpr, EmptyBindingSet.getInstance()); + return strategy.evaluate(tupleExpr, bindings); } catch (QueryEvaluationException e) { throw new SailException(e); @@ -239,16 +249,11 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I @Override protected CloseableIteration getContextIDsInternal() throws SailException { try { - return new FilterIteration(new ExceptionConvertingIteration(databaseConnection.listContexts()) { + return new ExceptionConvertingIteration(databaseConnection.listContexts()) { @Override protected SailException convert(Exception e) { return new SailException("database error while iterating over result set",e); } - }) { - @Override - protected boolean accept(Resource object) throws SailException { - return !object.stringValue().equals(defaultContext); - } }; } catch (SQLException e) { throw new SailException("database error while listing contexts",e); @@ -265,16 +270,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I contextSet.addAll(Lists.transform(Arrays.asList(contexts), new Function() { @Override public KiWiResource apply(Resource input) { - if(input == null) { - if(defaultContext != null) { - // null value for context means statements without context; in KiWi, this means "default context" - return (KiWiUriResource)valueFactory.createURI(defaultContext); - } else { - return null; - } - } else { - return valueFactory.convert(input); - } + return valueFactory.convert(input); } })); @@ -285,7 +281,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I @Override protected Iteration createIteration() throws RepositoryException { try { - return databaseConnection.listTriples(rsubj, rpred, robj, context, includeInferred, false); + return databaseConnection.listTriples(rsubj, rpred, robj, context, includeInferred); } catch (SQLException e) { throw new RepositoryException("database error while listing triples",e); } @@ -297,7 +293,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I @Override protected Iteration createIteration() throws RepositoryException { try { - return databaseConnection.listTriples(rsubj, rpred, robj, null, includeInferred, true); + return databaseConnection.listTriples(rsubj, rpred, robj, null, includeInferred); } catch (SQLException e) { throw new RepositoryException("database error while listing triples",e); } @@ -347,6 +343,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I // nothing to do, the database transaction is started automatically triplesAdded = false; triplesRemoved = false; + deletedStatementsLog = new HashSet(); } @Override @@ -357,6 +354,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I throw new SailException("database error while committing transaction",e); } if(triplesAdded || triplesRemoved) { + deletedStatementsLog.clear(); store.notifySailChanged(new SailChangedEvent() { @Override @@ -381,6 +379,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I protected void rollbackInternal() throws SailException { try { databaseConnection.rollback(); + deletedStatementsLog.clear(); } catch (SQLException e) { throw new SailException("database error while rolling back transaction",e); } @@ -395,9 +394,9 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I if(triple.getId() != null) { databaseConnection.deleteTriple(triple); triplesRemoved = true; + deletedStatementsLog.add(triple.getId()); notifyStatementRemoved(triple); } - valueFactory.removeStatement(triple); } triples.close(); } catch(SQLException ex) { @@ -426,9 +425,9 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I if(triple.getId() != null && triple.isInferred()) { databaseConnection.deleteTriple(triple); triplesRemoved = true; + deletedStatementsLog.add(triple.getId()); notifyStatementRemoved(triple); } - valueFactory.removeStatement(triple); } triples.close(); } catch(SQLException ex) { @@ -440,6 +439,12 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I /** * Removes an inferred statement from a specific context. * + * @param subj The subject of the statement that should be removed. + * @param pred The predicate of the statement that should be removed. + * @param obj The object of the statement that should be removed. + * @param contexts The context(s) from which to remove the statements. Note that this + * parameter is a vararg and as such is optional. If no contexts are + * supplied the method operates on the entire repository. * @throws org.openrdf.sail.SailException If the statement could not be removed. * @throws IllegalStateException If the connection has been closed. */ @@ -448,6 +453,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I if(triple.getId() != null && triple.isInferred()) { databaseConnection.deleteTriple(triple); triplesRemoved = true; + deletedStatementsLog.add(triple.getId()); notifyStatementRemoved(triple); } } catch(SQLException ex) { @@ -604,7 +610,7 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I throw new IllegalArgumentException("e must not be null"); } else { - throw new IllegalArgumentException("Unexpected exception type: " + e.getClass(),e); + throw new IllegalArgumentException("Unexpected exception type: " + e.getClass()); } } }; @@ -625,83 +631,6 @@ public class KiWiSailConnection extends NotifyingSailConnectionBase implements I } } - /** - * Return an iterator over the resources contained in this repository. - * - * @return - */ - @Override - public RepositoryResult getResources() throws RepositoryException { - try { - return new RepositoryResult(new ExceptionConvertingIteration(databaseConnection.listResources()) { - @Override - protected RepositoryException convert(Exception e) { - return new RepositoryException(e); - } - }); - } catch (SQLException e) { - throw new RepositoryException(e); - } - } - - /** - * Return an iterator over the resources contained in this repository matching the given prefix. - * - * @return - */ - @Override - public RepositoryResult getResources(String prefix) throws RepositoryException { - try { - return new RepositoryResult(new ExceptionConvertingIteration(databaseConnection.listResources(prefix)) { - @Override - protected RepositoryException convert(Exception e) { - return new RepositoryException(e); - } - }); - } catch (SQLException e) { - throw new RepositoryException(e); - } - } - - /** - * Return the Sesame URI with the given uri identifier if it exists, or null if it does not exist. - * - * @param uri - * @return - */ - @Override - public URI getURI(String uri) { - try { - return databaseConnection.loadUriResource(uri); - } catch (SQLException e) { - return null; - } - } - - /** - * Return the Sesame BNode with the given anonymous ID if it exists, or null if it does not exist. - * - * @param id - * @return - */ - @Override - public BNode getBNode(String id) { - try { - return databaseConnection.loadAnonResource(id); - } catch (SQLException e) { - return null; - } - } - - /** - * Remove the resource given as argument from the triple store and the resource repository. - * - * @param resource - */ - @Override - public void removeResource(Resource resource) { - // handled by garbage collection - } protected static class KiWiEvaluationStatistics extends EvaluationStatistics { http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java index 4119023..3c71e2c 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiStore.java @@ -18,7 +18,6 @@ package org.apache.marmotta.kiwi.sail; import com.google.common.collect.MapMaker; -import org.apache.marmotta.kiwi.config.KiWiConfiguration; import org.apache.marmotta.kiwi.model.caching.IntArray; import org.apache.marmotta.kiwi.persistence.KiWiDialect; import org.apache.marmotta.kiwi.persistence.KiWiPersistence; @@ -92,23 +91,20 @@ public class KiWiStore extends NotifyingSailBase { protected ConcurrentMap tripleRegistry; public KiWiStore(KiWiPersistence persistence, String defaultContext, String inferredContext) { - this.persistence = persistence; - this.defaultContext = defaultContext; + this.persistence = persistence; + this.defaultContext = defaultContext; this.nodeLock = new ReentrantLock(); this.tripleLock = new ReentrantLock(); this.inferredContext = inferredContext; - + + tripleRegistry = new MapMaker().weakValues().makeMap(); } - @Deprecated public KiWiStore(String name, String jdbcUrl, String db_user, String db_password, KiWiDialect dialect, String defaultContext, String inferredContext) { - this(new KiWiConfiguration(name,jdbcUrl,db_user,db_password,dialect, defaultContext, inferredContext)); + this(new KiWiPersistence(name,jdbcUrl,db_user,db_password,dialect), defaultContext, inferredContext); } - public KiWiStore(KiWiConfiguration configuration) { - this(new KiWiPersistence(configuration), configuration.getDefaultContext(), configuration.getInferredContext()); - } /** * Do store-specific operations to initialize the store. The default @@ -116,11 +112,9 @@ public class KiWiStore extends NotifyingSailBase { */ @Override protected void initializeInternal() throws SailException { - tripleRegistry = new MapMaker().weakValues().makeMap(); - try { - persistence.initialise(); persistence.initDatabase(); + persistence.initialise(); initialized = true; } catch (SQLException e) { @@ -174,19 +168,15 @@ public class KiWiStore extends NotifyingSailBase { protected void shutDownInternal() throws SailException { closeValueFactory(); persistence.shutdown(); - tripleRegistry = null; - initialized = false; } /** * In case there is a value factory managed by this repository directly, close it (and the underlying database * connection) */ - public synchronized void closeValueFactory() { + public void closeValueFactory() { if(repositoryValueFactory != null) { - repositoryValueFactory.close(); repositoryValueFactory = null; - persistence.setValueFactory(null); } } @@ -207,31 +197,11 @@ public class KiWiStore extends NotifyingSailBase { * @return a ValueFactory object for this Sail object. */ @Override - public synchronized ValueFactory getValueFactory() { + public ValueFactory getValueFactory() { if(repositoryValueFactory == null) { repositoryValueFactory = new KiWiValueFactory(this, defaultContext); - persistence.setValueFactory(repositoryValueFactory); } return repositoryValueFactory; } - /** - * Manually call the garbage collector for the triple store. Otherwise it will run every hour. - */ - public void garbageCollect() throws SailException { - try { - persistence.garbageCollect(); - } catch (SQLException e) { - throw new SailException("error calling garbage collector",e); - } - } - - - public boolean checkConsistency() throws SailException { - try { - return persistence.checkConsistency(); - } catch (SQLException e) { - throw new SailException("error calling consistency check",e); - } - } } http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java index 9ec70fb..22038f6 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java +++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/sail/KiWiValueFactory.java @@ -17,26 +17,35 @@ */ package org.apache.marmotta.kiwi.sail; -import org.apache.commons.lang3.LocaleUtils; -import org.apache.marmotta.commons.locking.ObjectLocks; +import com.google.common.collect.MapMaker; +import org.apache.commons.lang.LocaleUtils; import org.apache.marmotta.commons.sesame.model.LiteralCommons; -import org.apache.marmotta.commons.sesame.model.LiteralKey; import org.apache.marmotta.commons.sesame.model.Namespaces; import org.apache.marmotta.commons.util.DateUtils; import org.apache.marmotta.kiwi.model.caching.IntArray; import org.apache.marmotta.kiwi.model.rdf.*; import org.apache.marmotta.kiwi.persistence.KiWiConnection; -import org.openrdf.model.*; +import org.openrdf.model.BNode; +import org.openrdf.model.Literal; +import org.openrdf.model.Resource; +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.ValueFactory; import org.openrdf.model.impl.ContextStatementImpl; +import org.openrdf.model.impl.StatementImpl; +import org.openrdf.repository.RepositoryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.datatype.XMLGregorianCalendar; import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Random; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; /** * Add file description here! @@ -49,6 +58,7 @@ public class KiWiValueFactory implements ValueFactory { private Random anonIdGenerator; + /** * This is a hash map for storing references to resources that have not yet been persisted. It is used e.g. when * one or more transactions are currently active and request the creation of same resource several times @@ -66,88 +76,75 @@ public class KiWiValueFactory implements ValueFactory { private KiWiStore store; + private ReentrantLock nodeLock; + private ReentrantLock tripleLock; - private ObjectLocks resourceLocks; - private ObjectLocks literalLocks; + private ConcurrentMap resourceLocks; + private ConcurrentMap literalLocks; private String defaultContext; - private boolean batchCommit; - - private int batchSize = 1000; - - // the list containing the in-memory nodes that need to be committed later - private List nodeBatch; - - // a quick lookup allowing to lookup nodes while they are not yet in the database - private Map batchUriLookup; - private Map batchBNodeLookup; - private Map batchLiteralLookup; - - private int poolSize = 4; - private int poolPosition = 0; - - private ArrayList pooledConnections; - - - private ReentrantReadWriteLock commitLock = new ReentrantReadWriteLock(); - public KiWiValueFactory(KiWiStore store, String defaultContext) { - resourceLocks = new ObjectLocks(); - literalLocks = new ObjectLocks(); + nodeLock = store.nodeLock; + tripleLock = store.tripleLock; + resourceLocks = new MapMaker().weakKeys().weakValues().makeMap(); + literalLocks = new MapMaker().weakKeys().weakValues().makeMap(); anonIdGenerator = new Random(); tripleRegistry = store.tripleRegistry; this.store = store; this.defaultContext = defaultContext; - - // batch commits - this.nodeBatch = Collections.synchronizedList(new ArrayList(batchSize)); - - this.batchCommit = store.getPersistence().getConfiguration().isBatchCommit(); - this.batchSize = store.getPersistence().getConfiguration().getBatchSize(); - - this.batchUriLookup = new ConcurrentHashMap(); - this.batchBNodeLookup = new ConcurrentHashMap(); - this.batchLiteralLookup = new ConcurrentHashMap(); - - this.pooledConnections = new ArrayList<>(poolSize); - try { - for(int i = 0; i= batchSize) { - flushBatch(); - } - } catch (SQLException e) { - log.error("database error, could not load URI resource",e); - throw new IllegalStateException("database error, could not load URI resource",e); - } + return result; + } catch (SQLException e) { + log.error("database error, could not load URI resource",e); + throw new IllegalStateException("database error, could not load URI resource",e); + } finally { + releaseConnection(connection); + lock.unlock(); } - } /** @@ -252,56 +213,25 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public BNode createBNode(String nodeID) { - commitLock.readLock().lock(); - resourceLocks.lock(nodeID); - + nodeLock.lock(); + KiWiConnection connection = aqcuireConnection(); try { - KiWiAnonResource result = batchBNodeLookup.get(nodeID); + // first look in the registry for newly created resources if the resource has already been created and + // is still volatile + KiWiAnonResource result = connection.loadAnonResource(nodeID); - if(result != null) { - return result; - } else { - KiWiConnection connection = aqcuireConnection(); - try { - // first look in the registry for newly created resources if the resource has already been created and - // is still volatile - result = connection.loadAnonResource(nodeID); - - if(result == null) { - result = new KiWiAnonResource(nodeID); - - if(batchCommit) { - result.setId(connection.getNodeId()); - nodeBatch.add(result); - batchBNodeLookup.put(nodeID,result); - } else { - connection.storeNode(result, false); - } - } - if(result.getId() == null) { - log.error("node ID is null!"); - } - - return result; - } catch (SQLException e) { - log.error("database error, could not load anonymous resource",e); - throw new IllegalStateException("database error, could not load anonymous resource",e); - } finally { - releaseConnection(connection); - } + if(result == null) { + result = new KiWiAnonResource(nodeID); + connection.storeNode(result); } - } finally { - resourceLocks.unlock(nodeID); - commitLock.readLock().unlock(); - try { - if(nodeBatch.size() >= batchSize) { - flushBatch(); - } - } catch (SQLException e) { - log.error("database error, could not load URI resource",e); - throw new IllegalStateException("database error, could not load URI resource",e); - } + return result; + } catch (SQLException e) { + log.error("database error, could not load anonymous resource",e); + throw new IllegalStateException("database error, could not load anonymous resource",e); + } finally { + releaseConnection(connection); + nodeLock.unlock(); } } @@ -330,11 +260,12 @@ public class KiWiValueFactory implements ValueFactory { * @return a typed literal representation of the supplied object. * @since 2.7.0 */ + @Override public Literal createLiteral(Object object) { if(object instanceof XMLGregorianCalendar) { return createLiteral((XMLGregorianCalendar)object); } else { - return createLiteral(object,null,LiteralCommons.getXSDType(object.getClass())); + return createLiteral(object,null,null); } } @@ -345,9 +276,7 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public Literal createLiteral(String label) { - // FIXME: MARMOTTA-39 (no default datatype before RDF-1.1) - // return createLiteral(label, null, LiteralCommons.getXSDType(String.class)); - return createLiteral(label, null, null); + return createLiteral(label, null, LiteralCommons.getXSDType(String.class)); } /** @@ -359,9 +288,7 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public Literal createLiteral(String label, String language) { - // FIXME: MARMOTTA-39 (no rdf:langString before RDF-1.1) - // return createLiteral(label,language,LiteralCommons.getRDFLangStringType()); - return createLiteral(label, language, null); + return createLiteral(label,language,LiteralCommons.getRDFLangStringType()); } /** @@ -388,173 +315,112 @@ public class KiWiValueFactory implements ValueFactory { * @return */ private KiWiLiteral createLiteral(T value, String lang, String type) { - commitLock.readLock().lock(); - Locale locale; - if(lang != null) { - try { - Locale.Builder builder = new Locale.Builder(); - builder.setLanguageTag(lang); - locale = builder.build(); - } catch (IllformedLocaleException ex) { - log.warn("malformed language literal (language: {})", lang); - locale = null; - lang = null; - } - } else { - locale = null; - } - if (lang != null) { - // FIXME: MARMOTTA-39 (no rdf:langString) - // type = LiteralCommons.getRDFLangStringType(); - } else if (type == null) { - // FIXME: MARMOTTA-39 (no default datatype before RDF-1.1) - // type = LiteralCommons.getXSDType(value.getClass()); + type = LiteralCommons.getRDFLangStringType(); + } else if(type == null) { + type = LiteralCommons.getXSDType(value.getClass()); } - String key = LiteralCommons.createCacheKey(value.toString(),locale,type); - LiteralKey lkey = new LiteralKey(value,type,lang); - literalLocks.lock(lkey); + KiWiLiteral result = null; + + final KiWiUriResource rtype = (KiWiUriResource)createURI(type); + final Locale locale; + if(lang != null) { + locale = LocaleUtils.toLocale(lang); + } else + locale = null; + ReentrantLock lock = acquireLiteralLock(value); + KiWiConnection connection = aqcuireConnection(); try { - KiWiLiteral result = batchLiteralLookup.get(key); - if(result != null) { - return result; + // differentiate between the different types of the value + if(value instanceof Date || type.equals(Namespaces.NS_XSD+"dateTime")) { + // parse if necessary + final Date dvalue; + if(value instanceof Date) { + dvalue = (Date)value; + } else { + dvalue = DateUtils.parseDate(value.toString()); + } + + result = connection.loadLiteral(dvalue); + + if(result == null) { + result= new KiWiDateLiteral(dvalue, rtype); + } + } else if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass()) || + Long.class.equals(value.getClass()) || long.class.equals(value.getClass()) || + type.equals(Namespaces.NS_XSD+"integer") || type.equals(Namespaces.NS_XSD+"long")) { + long ivalue = 0; + if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass())) { + ivalue = (Integer)value; + } else if(Long.class.equals(value.getClass()) || long.class.equals(value.getClass())) { + ivalue = (Long)value; + } else { + ivalue = Long.parseLong(value.toString()); + } + + + result = connection.loadLiteral(ivalue); + + if(result == null) { + result= new KiWiIntLiteral(ivalue, rtype); + } + } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass()) || + Float.class.equals(value.getClass()) || float.class.equals(value.getClass()) || + type.equals(Namespaces.NS_XSD+"double") || type.equals(Namespaces.NS_XSD+"float")) { + double dvalue = 0.0; + if(Float.class.equals(value.getClass()) || float.class.equals(value.getClass())) { + dvalue = (Float)value; + } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass())) { + dvalue = (Double)value; + } else { + dvalue = Double.parseDouble(value.toString()); + } + + + result = connection.loadLiteral(dvalue); + + if(result == null) { + result= new KiWiDoubleLiteral(dvalue, rtype); + } + } else if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass()) || + type.equals(Namespaces.NS_XSD+"boolean")) { + boolean bvalue = false; + if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass())) { + bvalue = (Boolean)value; + } else { + bvalue = Boolean.parseBoolean(value.toString()); + } + + + result = connection.loadLiteral(bvalue); + + if(result == null) { + result= new KiWiBooleanLiteral(bvalue, rtype); + } } else { - final KiWiUriResource rtype = type==null?null:(KiWiUriResource)createURI(type); - - final KiWiConnection connection = aqcuireConnection(); - try { - - try { - // differentiate between the different types of the value - if (type == null) { - // FIXME: MARMOTTA-39 (this is to avoid a NullPointerException in the following if-clauses) - result = connection.loadLiteral(value.toString(), lang, rtype); - - if(result == null) { - result = new KiWiStringLiteral(value.toString(), locale, rtype); - } - } else if(value instanceof Date || type.equals(Namespaces.NS_XSD+"dateTime")) { - // parse if necessary - final Date dvalue; - if(value instanceof Date) { - dvalue = (Date)value; - } else { - dvalue = DateUtils.parseDate(value.toString()); - } - - result = connection.loadLiteral(dvalue); - - if(result == null) { - result= new KiWiDateLiteral(dvalue, rtype); - } - } else if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass()) || - Long.class.equals(value.getClass()) || long.class.equals(value.getClass()) || - type.equals(Namespaces.NS_XSD+"integer") || type.equals(Namespaces.NS_XSD+"long")) { - long ivalue = 0; - if(Integer.class.equals(value.getClass()) || int.class.equals(value.getClass())) { - ivalue = (Integer)value; - } else if(Long.class.equals(value.getClass()) || long.class.equals(value.getClass())) { - ivalue = (Long)value; - } else { - ivalue = Long.parseLong(value.toString()); - } - - - result = connection.loadLiteral(ivalue); - - if(result == null) { - result= new KiWiIntLiteral(ivalue, rtype); - } - } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass()) || - Float.class.equals(value.getClass()) || float.class.equals(value.getClass()) || - type.equals(Namespaces.NS_XSD+"double") || type.equals(Namespaces.NS_XSD+"float")) { - double dvalue = 0.0; - if(Float.class.equals(value.getClass()) || float.class.equals(value.getClass())) { - dvalue = (Float)value; - } else if(Double.class.equals(value.getClass()) || double.class.equals(value.getClass())) { - dvalue = (Double)value; - } else { - dvalue = Double.parseDouble(value.toString()); - } - - - result = connection.loadLiteral(dvalue); - - if(result == null) { - result= new KiWiDoubleLiteral(dvalue, rtype); - } - } else if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass()) || - type.equals(Namespaces.NS_XSD+"boolean")) { - boolean bvalue = false; - if(Boolean.class.equals(value.getClass()) || boolean.class.equals(value.getClass())) { - bvalue = (Boolean)value; - } else { - bvalue = Boolean.parseBoolean(value.toString()); - } - - - result = connection.loadLiteral(bvalue); - - if(result == null) { - result= new KiWiBooleanLiteral(bvalue, rtype); - } - } else { - result = connection.loadLiteral(value.toString(), lang, rtype); - - if(result == null) { - result = new KiWiStringLiteral(value.toString(), locale, rtype); - } - } - } catch(IllegalArgumentException ex) { - // malformed number or date - log.warn("malformed argument for typed literal of type {}: {}", rtype.stringValue(), value); - KiWiUriResource mytype = (KiWiUriResource)createURI(Namespaces.NS_XSD+"string"); - - result = connection.loadLiteral(value.toString(), lang, mytype); - - if(result == null) { - result = new KiWiStringLiteral(value.toString(), locale, mytype); - } - - } - - if(result.getId() == null) { - if(batchCommit) { - result.setId(connection.getNodeId()); - batchLiteralLookup.put(key, result); - - nodeBatch.add(result); - } else { - connection.storeNode(result, false); - } - } - - return result; - - - } catch (SQLException e) { - log.error("database error, could not load literal",e); - throw new IllegalStateException("database error, could not load literal",e); - } finally { - releaseConnection(connection); + result = connection.loadLiteral(value.toString(), lang, rtype); + + if(result == null) { + result = new KiWiStringLiteral(value.toString(), locale, rtype); } } - } finally { - literalLocks.unlock(lkey); - commitLock.readLock().unlock(); - try { - if(nodeBatch.size() >= batchSize) { - flushBatch(); - } - } catch (SQLException e) { - log.error("database error, could not load URI resource",e); - throw new IllegalStateException("database error, could not load URI resource",e); + if(result.getId() == null) { + connection.storeNode(result); } + + return result; + + } catch (SQLException e) { + log.error("database error, could not load literal",e); + throw new IllegalStateException("database error, could not load literal",e); + } finally { + releaseConnection(connection); + lock.unlock(); } } @@ -666,11 +532,7 @@ public class KiWiValueFactory implements ValueFactory { */ @Override public Statement createStatement(Resource subject, URI predicate, Value object) { - if(defaultContext != null) { - return createStatement(subject, predicate, object, createURI(defaultContext)); - } else { - return createStatement(subject, predicate, object, null); - } + return createStatement(subject, predicate, object, createURI(defaultContext)); } /** @@ -686,6 +548,41 @@ public class KiWiValueFactory implements ValueFactory { @Override public Statement createStatement(Resource subject, URI predicate, Value object, Resource context) { return new ContextStatementImpl(subject,predicate,object,context); + /* + tripleLock.lock(); + KiWiConnection connection = aqcuireConnection(); + try { + IntArray cacheKey = IntArray.createSPOCKey(subject,predicate,object,context); + Statement result = tripleRegistry.get(cacheKey); + if(result == null || ((KiWiTriple)result).isDeleted()) { + KiWiResource ksubject = convert(subject); + KiWiUriResource kpredicate = convert(predicate); + KiWiNode kobject = convert(object); + KiWiResource kcontext = convert(context); + + // test if the triple already exists in the database; if yes, return it + List triples = connection.listTriples(ksubject,kpredicate,kobject,kcontext,true).asList(); + if(triples.size() == 1) { + result = triples.get(0); + } else { + result = new KiWiTriple(ksubject,kpredicate,kobject,kcontext); + ((KiWiTriple)result).setMarkedForReasoning(true); + } + + tripleRegistry.put(cacheKey,result); + } + return result; + } catch (SQLException e) { + log.error("database error, could not load triple", e); + throw new IllegalStateException("database error, could not load triple",e); + } catch (RepositoryException e) { + log.error("database error, could not load triple", e); + throw new IllegalStateException("database error, could not load triple",e); + } finally { + releaseConnection(connection); + tripleLock.unlock(); + } + */ } /** @@ -700,19 +597,22 @@ public class KiWiValueFactory implements ValueFactory { * @return The created statement. */ public Statement createStatement(Resource subject, URI predicate, Value object, Resource context, KiWiConnection connection) { - IntArray cacheKey = IntArray.createSPOCKey(subject, predicate, object, context); - KiWiTriple result = (KiWiTriple)tripleRegistry.get(cacheKey); + IntArray cacheKey = IntArray.createSPOCKey(subject,predicate,object,context); + Statement result = tripleRegistry.get(cacheKey); try { - if(result == null || result.isDeleted()) { + if(result == null || ((KiWiTriple)result).isDeleted()) { KiWiResource ksubject = convert(subject); KiWiUriResource kpredicate = convert(predicate); KiWiNode kobject = convert(object); KiWiResource kcontext = convert(context); - result = new KiWiTriple(ksubject,kpredicate,kobject,kcontext); - result.setId(connection.getTripleId(ksubject,kpredicate,kobject,kcontext,true)); - if(result.getId() == null) { - result.setMarkedForReasoning(true); + // test if the triple already exists in the database; if yes, return it + List triples = connection.listTriples(ksubject,kpredicate,kobject,kcontext,true).asList(); + if(triples.size() == 1) { + result = triples.get(0); + } else { + result = new KiWiTriple(ksubject,kpredicate,kobject,kcontext); + ((KiWiTriple)result).setMarkedForReasoning(true); } tripleRegistry.put(cacheKey,result); @@ -721,18 +621,12 @@ public class KiWiValueFactory implements ValueFactory { } catch (SQLException e) { log.error("database error, could not load triple", e); throw new IllegalStateException("database error, could not load triple",e); + } catch (RepositoryException e) { + log.error("database error, could not load triple", e); + throw new IllegalStateException("database error, could not load triple",e); } } - /** - * Remove a statement from the triple registry. Called when the statement is deleted and the transaction commits. - * @param triple - */ - protected void removeStatement(KiWiTriple triple) { - IntArray cacheKey = IntArray.createSPOCKey(triple.getSubject(), triple.getPredicate(), triple.getObject(), triple.getContext()); - tripleRegistry.remove(cacheKey); - triple.setDeleted(true); - } public KiWiResource convert(Resource r) { @@ -760,87 +654,4 @@ public class KiWiValueFactory implements ValueFactory { } } - - public boolean isBatchCommit() { - return batchCommit; - } - - public void setBatchCommit(boolean batchCommit) { - this.batchCommit = batchCommit; - } - - public int getBatchSize() { - return batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - - /** - * Immediately flush the batch to the database using the value factory's connection. The method expects the - * underlying connection to start and commit the node batch. - */ - public void flushBatch() throws SQLException { - KiWiConnection con = aqcuireConnection(); - try { - flushBatch(con); - } finally { - releaseConnection(con); - } - - } - - - /** - * Immediately flush the batch to the database. The method expects the underlying connection to start and commit - * the node batch. - */ - public void flushBatch(KiWiConnection con) throws SQLException { - commitLock.writeLock().lock(); - try { - if(batchCommit && nodeBatch.size() > 0) { - List processed = this.nodeBatch; - this.nodeBatch = Collections.synchronizedList(new ArrayList(batchSize)); - - con.startNodeBatch(); - - for(KiWiNode n : processed) { - con.storeNode(n,true); - } - batchLiteralLookup.clear(); - batchUriLookup.clear(); - batchBNodeLookup.clear(); - - con.commitNodeBatch(); - } - } finally { - commitLock.writeLock().unlock(); - } - } - - public void close() { - - for(KiWiConnection con : pooledConnections) { - try { - if(!con.isClosed()) { - if(batchCommit && nodeBatch.size() > 0) { - try { - flushBatch(con); - } catch (SQLException e) { - log.error("error while flushing node batch",e); - } - } - - con.commit(); - con.close(); - } - } catch (SQLException e) { - log.warn("could not close value factory connection: {}",e.getMessage()); - } - } - } - - } http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml index 9d27ef1..179f24d 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml @@ -149,7 +149,7 @@ are "on" and "off". The default is "autodetect". a cache from database ID to KiWiNode; should be very large since this kind of lookup is a very frequent operation --> @@ -159,7 +159,7 @@ are "on" and "off". The default is "autodetect". from the database by avoiding reconstructing each triple from the database result --> @@ -176,7 +176,7 @@ are "on" and "off". The default is "autodetect". http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql index 3493187..0e4fed5 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/create_base_tables.sql @@ -20,7 +20,7 @@ CREATE SEQUENCE seq_namespaces; CREATE TABLE nodes ( id bigint NOT NULL, ntype char(8) NOT NULL, - svalue varchar(2147483647) NOT NULL, + svalue varchar(65536) NOT NULL, dvalue double precision, ivalue bigint, tvalue timestamp, @@ -36,7 +36,7 @@ CREATE TABLE triples ( subject bigint NOT NULL REFERENCES nodes(id), predicate bigint NOT NULL REFERENCES nodes(id), object bigint NOT NULL REFERENCES nodes(id), - context bigint REFERENCES nodes(id), + context bigint NOT NULL REFERENCES nodes(id), creator bigint REFERENCES nodes(id), inferred boolean DEFAULT false, deleted boolean DEFAULT false, @@ -66,13 +66,18 @@ CREATE TABLE metadata ( CREATE INDEX idx_node_content ON nodes(svalue); CREATE INDEX idx_literal_lang ON nodes(lang); +CREATE INDEX idx_triples_s ON triples(subject); +CREATE INDEX idx_triples_o ON triples(object); +CREATE INDEX idx_triples_sp ON triples(subject,predicate); +CREATE INDEX idx_triples_po ON triples(predicate,object); CREATE INDEX idx_triples_spo ON triples(subject,predicate,object); -CREATE INDEX idx_triples_op ON triples(object,predicate); +CREATE INDEX idx_triples_cs ON triples(context,subject); +CREATE INDEX idx_triples_csp ON triples(context,subject,predicate); CREATE INDEX idx_triples_cspo ON triples(context,subject,predicate,object); CREATE INDEX idx_namespaces_uri ON namespaces(uri); CREATE INDEX idx_namespaces_prefix ON namespaces(prefix); -- insert initial metadata -INSERT INTO metadata(mkey,mvalue) VALUES ('version','2'); +INSERT INTO metadata(mkey,mvalue) VALUES ('version','1'); INSERT INTO metadata(mkey,mvalue) VALUES ('created',FORMATDATETIME(now(),'yyyy-MM-dd HH:mm:ss z','en') ); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/drop_base_tables.sql ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/drop_base_tables.sql b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/drop_base_tables.sql index 41ca5f7..bf3d7e4 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/drop_base_tables.sql +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/drop_base_tables.sql @@ -16,8 +16,13 @@ DROP INDEX IF EXISTS idx_node_content; DROP INDEX IF EXISTS idx_literal_lang; -DROP INDEX IF EXISTS idx_triples_op; +DROP INDEX IF EXISTS idx_triples_s; +DROP INDEX IF EXISTS idx_triples_o; +DROP INDEX IF EXISTS idx_triples_sp; +DROP INDEX IF EXISTS idx_triples_po; DROP INDEX IF EXISTS idx_triples_spo; +DROP INDEX IF EXISTS idx_triples_cs; +DROP INDEX IF EXISTS idx_triples_csp; DROP INDEX IF EXISTS idx_triples_cspo; DROP INDEX IF EXISTS idx_namespaces_uri; http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/4aa87dbe/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties ---------------------------------------------------------------------- diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties index 092cfcb..2fc9afc 100644 --- a/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties +++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/org/apache/marmotta/kiwi/persistence/h2/statements.properties @@ -36,10 +36,10 @@ load.literal_by_v = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang, load.literal_by_vl = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND lang = ? load.literal_by_vt = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE svalue = ? AND ltype = ? -load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ? -load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ? -load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ? -load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ? +load.literal_by_iv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE ivalue = ? AND lang IS NULL AND ltype = ? +load.literal_by_dv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE dvalue = ? AND lang IS NULL AND ltype = ? +load.literal_by_tv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE tvalue = ? AND lang IS NULL AND ltype = ? +load.literal_by_bv = SELECT id,ntype,svalue,ivalue,dvalue,tvalue,bvalue,lang,ltype,createdAt FROM nodes WHERE bvalue = ? AND lang IS NULL AND ltype = ? load.namespace_prefix = SELECT id,prefix,uri,createdAt FROM namespaces WHERE prefix = ?; @@ -60,22 +60,16 @@ store.tliteral = INSERT INTO nodes (id,ntype,svalue,tvalue,ltype,createdAt store.namespace = INSERT INTO namespaces (id,prefix,uri,createdAt) VALUES (?,?,?,?) -#store.triple = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt) VALUES (?,?,?,?,?,?,?) -store.triple = MERGE INTO triples (id,subject,predicate,object,context,inferred,createdAt) KEY(id) VALUES (?,?,?,?,?,?,?) -load.triple = SELECT id FROM triples WHERE subject = ? AND predicate = ? AND object = ? AND context = ? AND deleted = false AND inferred = true +store.triple = INSERT INTO triples (id,subject,predicate,object,context,inferred,createdAt) VALUES (?,?,?,?,?,?,?) -query.size = SELECT count(*) FROM triples WHERE deleted = false AND inferred = false -query.size_ctx = SELECT count(*) FROM triples WHERE context = ? AND deleted = false AND inferred = false +query.size = SELECT count(*) FROM triples WHERE deleted = false +query.size_ctx = SELECT count(*) FROM triples WHERE context = ? AND deleted = false query.contexts = SELECT DISTINCT context FROM triples WHERE deleted = false query.namespaces = SELECT id,prefix,uri,createdAt FROM namespaces -query.resources = SELECT id,ntype,svalue,createdAt FROM nodes WHERE ntype = 'uri' OR ntype = 'bnode' -query.resources_prefix = SELECT id,ntype,svalue,createdAt FROM nodes WHERE ntype = 'uri' AND svalue LIKE ? + # delete entities delete.triple = UPDATE triples SET deleted = true, deletedAt = now() WHERE id = ? undelete.triple = UPDATE triples SET deleted = false, deletedAt = NULL WHERE id = ? delete.namespace = DELETE FROM namespaces WHERE id = ? - -gc.check_consistency = SELECT svalue, ntype, count(id), max(id) FROM nodes group by svalue, ntype having count(id) > 1 -gc.list_node_ids = SELECT id FROM nodes WHERE svalue = ? AND ntype = ? AND id != ?