Return-Path: X-Original-To: apmail-jena-commits-archive@www.apache.org Delivered-To: apmail-jena-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 947DD1195C for ; Tue, 26 Aug 2014 20:30:34 +0000 (UTC) Received: (qmail 33223 invoked by uid 500); 26 Aug 2014 20:30:34 -0000 Delivered-To: apmail-jena-commits-archive@jena.apache.org Received: (qmail 33199 invoked by uid 500); 26 Aug 2014 20:30:34 -0000 Mailing-List: contact commits-help@jena.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@jena.apache.org Delivered-To: mailing list commits@jena.apache.org Received: (qmail 33190 invoked by uid 99); 26 Aug 2014 20:30:34 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Aug 2014 20:30:34 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 26 Aug 2014 20:30:32 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 4BA6B23889CB; Tue, 26 Aug 2014 20:30:12 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1620722 - in /jena/trunk/jena-arq: ./ src/main/java/org/apache/jena/riot/system/ Date: Tue, 26 Aug 2014 20:30:12 -0000 To: commits@jena.apache.org From: andy@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140826203012.4BA6B23889CB@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: andy Date: Tue Aug 26 20:30:11 2014 New Revision: 1620722 URL: http://svn.apache.org/r1620722 Log: More stream related classes Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDF.java jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDFText.java jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/StreamRDFCollectOne.java Modified: jena/trunk/jena-arq/ReleaseNotes.txt Modified: jena/trunk/jena-arq/ReleaseNotes.txt URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/ReleaseNotes.txt?rev=1620722&r1=1620721&r2=1620722&view=diff ============================================================================== --- jena/trunk/jena-arq/ReleaseNotes.txt (original) +++ jena/trunk/jena-arq/ReleaseNotes.txt Tue Aug 26 20:30:11 2014 @@ -2,6 +2,16 @@ ChangeLog for ARQ ================= (the list covers new features and refinements. See ASF JIRA for details of bugs fixed) +==== Jena 2.12.1 + ++ JENA-768 : Add "--out" to the "riot" command to choose a + different output format. ++ JENA-769 : StreamWriter, a place to get an output class that streams. + This supplements RDFDataMgr. ++ JENA-767 : Fix the JSON parser to handle \b escapes. ++ JENA-762 : Provide a registry for result set readers and writers. ++ JENA-761 : Provide a means to disable SERVICE + ==== Jena 2.12.0 Jena now requires Java7 as of this version. Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDF.java URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDF.java?rev=1620722&view=auto ============================================================================== --- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDF.java (added) +++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDF.java Tue Aug 26 20:30:11 2014 @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.riot.system; + +import org.apache.jena.atlas.iterator.IteratorSlotted ; +import org.apache.jena.riot.system.StreamRowRDF ; + +/** + * Pull version of processing StreamRDF + */ +public abstract class IteratorStreamRDF extends IteratorSlotted { } + Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDFText.java URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDFText.java?rev=1620722&view=auto ============================================================================== --- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDFText.java (added) +++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/IteratorStreamRDFText.java Tue Aug 26 20:30:11 2014 @@ -0,0 +1,243 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.riot.system; + +import static org.apache.jena.riot.tokens.TokenType.DIRECTIVE ; +import static org.apache.jena.riot.tokens.TokenType.DOT ; +import static org.apache.jena.riot.tokens.TokenType.IRI ; +import static org.apache.jena.riot.tokens.TokenType.PREFIXED_NAME ; + +import java.io.InputStream ; +import java.util.* ; + +import org.apache.jena.atlas.lib.Closeable ; +import org.apache.jena.atlas.lib.NotImplemented ; +import org.apache.jena.riot.RiotException ; +import org.apache.jena.riot.system.RiotLib ; +import org.apache.jena.riot.system.StreamRowRDF ; +import org.apache.jena.riot.tokens.Token ; +import org.apache.jena.riot.tokens.Tokenizer ; +import org.apache.jena.riot.tokens.TokenizerFactory ; +import org.slf4j.Logger ; +import org.slf4j.LoggerFactory ; + +import com.hp.hpl.jena.graph.Node ; + +/** Testing/development convenience. + * Iterator of StreamRowRDF (always a tuple) for an input stream of tokenized RDT terms. + */ +public class IteratorStreamRDFText extends IteratorStreamRDF implements Iterator { + private final TokenInputStream in ; + private Node[] previousTuple = null ; + + private /*public*/ IteratorStreamRDFText(InputStream input) { + Tokenizer t = TokenizerFactory.makeTokenizerUTF8(input) ; + in = new TokenInputStream(null, t) ; + } + + @Override + protected boolean hasMore() { + return true ; + } + + @Override + protected StreamRowRDF moveToNext() { + if ( ! in.hasNext() ) return null ; + List line = in.next() ; + StreamRowRDF row = line2row(line) ; + return row ; + } + + private StreamRowRDF line2row(List line) { + if ( line.size() != 3 && line.size() != 4 ) + throw new RiotException("Input line is not 3 or 4 items long") ; + + Node[] tuple = new Node[line.size()] ; + int idx = 0 ; + for ( Token token : line ) { + Node n = null ; + if ( ( token.isWord() && token.getImage().equals("R") ) + //|| ( token.isCtlCode() && token.getCntrlCode() == -1 ) // * + ) { + if ( previousTuple == null ) + throw new RiotException("Repeat without previous data row") ; + if ( idx >= previousTuple.length) + throw new RiotException("Repeat position beyond previous data row") ; + n = previousTuple[idx] ; + } else if ( token.isNode() ) { + n = asNode(token) ; + } + if ( n == null ) + throw new RiotException("Unrecognized token : "+token ) ; + tuple[idx] = n ; + idx++ ; + } + previousTuple = tuple ; + + // Needs rethink. + throw new NotImplemented() ; + +// if ( line.size() == 3 ) +// return new StreamRowRDFBase(Triple.create(tuple[0], tuple[1], tuple[2])) ; +// else +// return new StreamRowRDFBase(Quad.create(tuple[0], tuple[1], tuple[2], tuple[3])) ; +// return new StreamRowRDFBase(Tuple.create(tuple)) ; + } + + private static Node asNode(Token t) { + // <_:...> bnodes. + if ( t.isIRI() ) + return RiotLib.createIRIorBNode(t.getImage()) ; + return t.asNode() ; + } + + /** Tokenizer that sorts out prefixes and groups into sequences of token */ + private static class TokenInputStream implements Iterator>, Iterable>, Closeable { + private static Logger log = LoggerFactory.getLogger(TokenInputStream.class) ; + private boolean finished = false ; + private final Tokenizer tokens ; + private List list ; + private Map map = new HashMap() ; + private String label ; + + public TokenInputStream(String label, Tokenizer tokens) { + this.tokens = tokens ; + this.label = label ; + } + + @Override + public boolean hasNext() { + if ( finished ) + return false ; + + if ( list != null ) // Already got the reply. + return true ; + + try { + if ( !tokens.hasNext() ) { + finished = true ; + return false ; + } + list = buildOneLine() ; + if ( false && log.isDebugEnabled() ) + log.debug("Tokens: " + list) ; + if ( list == null ) + finished = true ; + return list != null ; + } catch (Exception ex) { + finished = true ; + return false ; + } + } + + private List buildOneLine() { + List tuple = new ArrayList() ; + boolean isDirective = false ; + for (; tokens.hasNext();) { + Token token = tokens.next() ; + + if ( token.hasType(DIRECTIVE) ) + isDirective = true ; + + if ( token.hasType(DOT) ) { + if ( tuple.size() > 0 && tuple.get(0).hasType(DIRECTIVE) ) { + directive(tuple) ; + tuple.clear() ; + isDirective = false ; + // Start again. + continue ; + } + return tuple ; + } + + // Fixup prefix names. + if ( !isDirective && token.hasType(PREFIXED_NAME) ) { + String ns = map.get(token.getImage()) ; + String iri ; + if ( ns == null ) { + log.warn("Can't resolve '" + token.toString(false) + "'", ns) ; + iri = "unresolved:" + token.getImage() + ":" + token.getImage2() ; + } else + iri = ns + token.getImage2() ; + token.setType(IRI) ; + token.setImage(iri) ; + token.setImage2(null) ; + } + + tuple.add(token) ; + } + + // No final DOT + return tuple ; + } + + private void directive(List tuple) { + if ( tuple.size() != 3 ) + throw new RiotException("Bad directive: " + tuple) ; + + String x = tuple.get(0).getImage() ; + + if ( x.equals("prefix") ) { + // Raw - unresolved prefix name. + if ( !tuple.get(1).hasType(PREFIXED_NAME) ) + throw new RiotException("@prefix requires a prefix (found '" + tuple.get(1) + "')") ; + if ( tuple.get(1).getImage2().length() != 0 ) + throw new RiotException("@prefix requires a prefix and no suffix (found '" + tuple.get(1) + "')") ; + String prefix = tuple.get(1).getImage() ; + + if ( !tuple.get(2).hasType(IRI) ) + throw new RiotException("@prefix requires an IRI (found '" + tuple.get(1) + "')") ; + String iriStr = tuple.get(2).getImage() ; + map.put(prefix, iriStr) ; + return ; + } + throw new RiotException("Unregcognized directive: " + x) ; + } + + @Override + public List next() { + if ( !hasNext() ) + throw new NoSuchElementException() ; + List r = list ; + if ( log.isDebugEnabled() ) { + if ( label != null ) + log.debug("<< " + label + ": " + r) ; + else + log.debug("<< " + r.toString()) ; + } + list = null ; + return r ; + } + + @Override + public void remove() { + throw new UnsupportedOperationException() ; + } + + @Override + public Iterator> iterator() { + return this ; + } + + @Override + public void close() {} + } + +} + Added: jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/StreamRDFCollectOne.java URL: http://svn.apache.org/viewvc/jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/StreamRDFCollectOne.java?rev=1620722&view=auto ============================================================================== --- jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/StreamRDFCollectOne.java (added) +++ jena/trunk/jena-arq/src/main/java/org/apache/jena/riot/system/StreamRDFCollectOne.java Tue Aug 26 20:30:11 2014 @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.riot.system; + +import com.hp.hpl.jena.graph.Triple ; +import com.hp.hpl.jena.sparql.core.Quad ; + +/** Take one item from a StreamRDF and present as a StreamRowRDF + * @see StreamRDF + * @see StreamRowRDF + */ +public class StreamRDFCollectOne implements StreamRDF { + StreamRowRDF row = null ; + private final PrefixMap pmap ; + public StreamRDFCollectOne() { + pmap = PrefixMapFactory.create() ; + } + + public StreamRDFCollectOne(PrefixMap pmap) { + this.pmap = pmap ; + } + + @Override + public void start() {} + + @Override + public void triple(Triple triple) { + row = new StreamRowRDFBase(triple) ; + } + + @Override + public void quad(Quad quad) { + row = new StreamRowRDFBase(quad) ; + } + + @Override + public void base(String base) {} + + @Override + public void prefix(String prefix, String iri) { + row = null ; + pmap.add(prefix, iri) ; + } + + @Override + public void finish() {} + + public StreamRowRDF getRow() { + return row ; + } +}