jena-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [47/50] [abbrv] jena git commit: JENA-1396: Merge //github/afs/mantis as subdirectory jena-db/
Date Thu, 28 Sep 2017 11:09:03 GMT
http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/MetaFile.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/MetaFile.java
index 0000000,0000000..bbe3221
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/MetaFile.java
@@@ -1,0 -1,0 +1,364 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import java.io.* ;
++import java.util.* ;
++
++import org.apache.jena.atlas.lib.* ;
++import org.apache.jena.atlas.lib.Closeable ;
++import org.seaborne.dboe.DBOpEnvException ;
++import org.seaborne.dboe.sys.Names ;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++/** Abstraction and many convenience operations on metadata. 
++ * Metadata is recorded in Java properties style - not RDF - because it's relative to the file or context used.
++ * Keys and values are always strings.
++ */
++public class MetaFile implements Sync, Closeable
++{
++    private static Comparator<String> comparator = new ComparatorKeys() ;
++    private static Logger log = LoggerFactory.getLogger(MetaFile.class) ;
++    private String metaFilename = null ;
++    private Properties properties = null ;
++    private String label = null ;
++    private boolean changed = false ;
++    private boolean closed = false ;
++    
++    /** Create a MetaFile
++     *  
++     * @param label     Convenience label.
++     * @param fn        On disk filename; {@link Names#memName} for in-memory
++     */
++    public MetaFile(String label, String fn)
++    {
++        this.label = label ;
++        this.metaFilename = fn ;
++
++        if ( fn == null || Names.isMem(fn) )
++            // In-memory.
++            return ;
++        
++        // Make absolute (current directory may change later)
++        if ( ! fn.endsWith(Names.extMeta) )
++            fn = fn+"."+Names.extMeta ;
++        File f = new File(fn) ;
++        this.metaFilename = f.getAbsolutePath() ;
++        // Does not load the details yet.
++        // JDI
++        ensureInit() ; 
++    }
++    
++    private void ensureInit()
++    { 
++        if ( properties == null )
++        {
++            properties = new PropertiesSorted(comparator) ;
++            if ( metaFilename != null )
++                loadProperties() ;
++        }
++    }
++    
++    /** Does this metafile exist on disk? (In-memory MetaFiles always exist) */
++    public boolean existsMetaData()
++    {
++        if ( isMem() )
++            return true ;
++        File f = new File(metaFilename) ;
++        if ( f.isDirectory() )
++            log.warn("Metadata file clashes with a directory") ;
++        return f.exists() && f.isFile() ;
++    }
++    
++    public String getFilename()         { return metaFilename ; } 
++
++    /** Test for the presence of a property */
++    public boolean hasProperty(String key)
++    {
++        return _getProperty(key, null) != null ;
++    }
++
++    /** Get the property value or null. */
++    public String getProperty(String key)
++    {
++        return _getProperty(key, null) ;
++    }
++    
++    /** Get the property value or return supplied default. */
++    public String getProperty(String key, String defaultString)
++    {
++        return _getProperty(key, defaultString) ;
++    }
++
++    /** Get the property value and parse as an integer */
++    public int getPropertyAsInteger(String key)
++    {
++        return Integer.parseInt(_getProperty(key, null)) ;
++    }
++    
++    /** Get the property value and parse as an integer or return default value. */
++    public int getPropertyAsInteger(String key, int defaultValue)
++    {
++        String x = getProperty(key) ;
++        if ( x == null )
++            return defaultValue ;
++        return Integer.parseInt(x) ;
++    }
++
++    /** Get property as a string and split on ",". */
++    public String[] getPropertySplit(String key)
++    {
++        String str = getProperty(key) ;
++        if ( str == null )
++            return null ;
++        return str.split(",") ;
++    }
++    
++    /** Get property as a string and split on ",", using the default string if not present in the MetaFile. */
++    public String[] getPropertySplit(String key, String defaultString)
++    {
++        String str = getProperty(key, defaultString) ;
++        return str.split(",") ;
++    }
++    
++    /** Set property */
++    public void setProperty(String key, String value)
++    {
++        _setProperty(key, value) ;
++    }
++    
++    /** Set property, turning integer into a string. */
++    public void setProperty(String key, int value)
++    {
++        _setProperty(key, Integer.toString(value)) ;
++    }
++    
++    /** Test whether a property has a value.  Null tests equal to not present. */
++    public boolean propertyEquals(String key, String value)
++    {
++        return Objects.equals(getProperty(key), value) ;
++    }
++
++    /** Set property if not already set. */
++    public void ensurePropertySet(String key, String expected)
++    {
++        getOrSetDefault(key, expected) ;
++    }
++
++    /** Get property or the default value - also set the default value if not present */
++    public String getOrSetDefault(String key, String expected)
++    {
++        String x = getProperty(key) ;
++        if ( x == null )
++        {
++            setProperty(key, expected) ;
++            x = expected ;
++        }
++        return x ;
++    }
++    
++    /** Check property is an expected value or set if missing */
++    public void checkOrSetMetadata(String key, String expected)
++    {
++        String x = getProperty(key) ;
++        if ( x == null )
++        {
++            setProperty(key, expected) ;
++            return ; 
++        }
++        if ( x.equals(expected) )
++            return ;
++        
++        inconsistent(key, x, expected) ; 
++    }
++
++    /** Check property has the value given - throw exception if not. */
++    public void checkMetadata(String key, String expected)
++    {
++        String value = getProperty(key) ;
++        
++        if ( !Objects.equals(value, value) )
++            inconsistent(key, value, expected) ;
++    }
++
++    private static void inconsistent(String key, String actual, String expected) 
++    {
++        String msg = String.format("Inconsistent: key=%s value=%s expected=%s", 
++                                   key, 
++                                   (actual==null?"<null>":actual),
++                                   (expected==null?"<null>":expected) ) ;
++        throw new MetaFileException(msg) ; 
++    }
++    
++    /** Clear all properties. */
++    public void clear()
++    {
++        _clear() ;
++    }
++
++    // ---- All get/set access through these  operations
++    private String _getProperty(String key, String dft)
++    {
++        ensureInit() ;
++        return properties.getProperty(key, dft) ;
++    }
++    
++    private void _setProperty(String key, String value)
++    {
++        ensureInit() ;
++        properties.setProperty(key, value) ;
++        changedEvent() ;
++    }
++    
++    /** Clear all properties. */
++    private void _clear()
++    {
++        ensureInit() ;
++        properties.clear() ;
++        changedEvent() ;
++    }
++
++    private void changedEvent() { changed = true ; }
++    // ----
++    
++    private boolean isMem() { return Names.isMem(metaFilename) ; }
++    
++    /** Write to backing file if changed */
++    public void flush()
++    {
++        if ( log.isDebugEnabled() )
++            log.debug("Flush metadata ("+changed+"): "+this.label) ;
++        if ( ! changed )
++            return ;
++        
++        
++        if ( log.isDebugEnabled() )
++        {
++            ByteArrayOutputStream out = new ByteArrayOutputStream() ;
++            PrintStream ps = new PrintStream(out) ;
++            properties.list(ps) ;
++            ps.flush() ;
++            log.debug("\n"+out.toString()) ;
++        }
++        
++        //properties.list(System.out) ;
++        
++        saveProperties() ;
++        changed = false ;
++    }
++
++    private void saveProperties()
++    {
++        if ( isMem() )
++            return ;
++        String str = label ;
++        if ( str == null )
++            str = metaFilename ;
++        str = "Metadata: "+str ;
++
++        try {
++            PropertyUtils.storeToFile(properties, str, metaFilename) ;
++        } 
++        catch (IOException ex)
++        {
++            log.error("Failed to store properties: "+metaFilename, ex) ;
++        }
++    }
++
++    
++    private void loadProperties()
++    {
++        if ( isMem() )
++        {
++            properties = new Properties() ;
++            return ;
++        }
++        
++        if ( properties == null )
++            properties = new Properties() ;
++        
++        // if ( metaFilename == null )
++        InputStream in = null ;
++        try { 
++            //  Copes with UTF-8 for Java5. 
++            PropertyUtils.loadFromFile(properties, metaFilename) ;
++        }
++        catch (FileNotFoundException ex) {} 
++        catch (IOException ex)
++        {
++            log.error("Failed to load properties: "+metaFilename, ex) ;
++        }
++    }
++    
++    /** Debugging */
++    public void dump(PrintStream output)
++    {
++        output.println("Metafile: "+metaFilename) ;
++        output.println("Label: "+label) ;
++        output.println("Status: "+(changed?"changed":"unchanged")) ;
++        
++        if ( properties == null )
++        {
++            output.println("#<null>") ;
++            return ;
++        }
++        // properties.list() ;
++        SortedSet<Object> x = new TreeSet<>() ;
++        x.addAll(properties.keySet()) ;
++        
++        for ( Object k : x )
++        {
++            String key = (String)k ;
++            String value = properties.getProperty(key) ;
++            output.print(key) ;
++            output.print("=") ;
++            output.print(value) ;
++            output.println() ;
++        }
++    }
++
++    @Override
++    public void sync()                  { flush() ; }
++    
++    @Override
++    public void close()
++    {
++        flush() ;
++        closed = true ;
++        metaFilename = null ;
++        properties = null ;
++
++    }
++
++    private static class ComparatorKeys implements Comparator<String>
++    {
++        @Override
++        public int compare(String o1, String o2)
++        {
++            return - o1.compareTo(o2) ;
++        }
++        
++    }
++    
++    private static class MetaFileException extends DBOpEnvException
++    {
++        MetaFileException(String msg) { super(msg) ; }
++    }
++
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/ProcessFileLock.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/ProcessFileLock.java
index 0000000,0000000..990dcab
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/ProcessFileLock.java
@@@ -1,0 -1,0 +1,215 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements.  See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership.  The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License.  You may obtain a copy of the License at
++ *
++ *     http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import static java.nio.file.StandardOpenOption.CREATE;
++import static java.nio.file.StandardOpenOption.DSYNC;
++import static java.nio.file.StandardOpenOption.READ;
++import static java.nio.file.StandardOpenOption.WRITE;
++
++import java.io.FileNotFoundException;
++import java.io.IOException;
++import java.nio.ByteBuffer;
++import java.nio.channels.FileChannel;
++import java.nio.channels.FileLock;
++import java.nio.file.NoSuchFileException;
++import java.nio.file.Path;
++import java.nio.file.Paths;
++import java.util.concurrent.ConcurrentHashMap;
++
++import org.apache.jena.atlas.RuntimeIOException;
++import org.apache.jena.atlas.io.IO;
++import org.apache.jena.atlas.lib.StrUtils;
++import org.apache.jena.atlas.logging.Log;
++import org.seaborne.dboe.DBOpEnvException;
++import org.seaborne.dboe.sys.ProcessUtils;
++
++/** A simple packaging around a {@link java.nio.channels.FileLock}. 
++ *  {@code ProcessFileLock} are not reentrant.
++ */
++public class ProcessFileLock {
++
++    // Static (process-wide) sync.
++    private static Object sync = new Object();
++    
++    // Map from path of the file to a ProcessFileLock
++    private static ConcurrentHashMap<Path, ProcessFileLock> locks =  new ConcurrentHashMap<>();
++
++    private final Path filepath;
++    private final FileChannel fileChannel;
++    private FileLock fileLock;
++
++    // Different variations for what to do when a lock can not be taken.
++    private enum NoLockAction { EXCEPTION, RETURN, WAIT }
++
++    /** Create a {@code ProcessFileLock} using the named file.
++     *  Locks are JVM-wide; each filename is assocauted with one lock object.  
++     */
++    public static ProcessFileLock create(String filename) {
++        try {
++            Path abspath = Paths.get(filename).toRealPath();
++            return locks.computeIfAbsent(abspath, ProcessFileLock::new);
++        }
++        catch (IOException e) { IO.exception(e); return null; }
++    }
++    
++    /** Return the lock, unlocking the file if this process has it locked. */
++    public static void release(ProcessFileLock lockFile) {
++        if ( lockFile == null )
++            return ;
++        if ( lockFile.isLockedHere() )
++            lockFile.unlock();
++        locks.remove(lockFile.getPath());
++    }
++    /** Create the structure for a ProcessFileLock on file {@code filename}.
++     * This does not take the lock
++     * 
++     * @see #lockEx()
++     * @see #tryLock()
++     * @see #unlock()
++     */
++    private ProcessFileLock(Path filename) {
++        try {
++            this.filepath = filename ;
++            // Much the same as ... 
++//            randomAccessFile = new RandomAccessFile(filename, "rw");
++//            fileChannel = randomAccessFile.getChannel();
++            // Quite heavy weight but only used to lock long-term objects.
++            this.fileChannel = FileChannel.open(filename, CREATE, WRITE, READ, DSYNC);
++            fileLock = null;
++        }
++        catch (NoSuchFileException | FileNotFoundException ex) {
++            // The path does not name a possible file in an exists directory.
++            throw new RuntimeIOException("No such file '"+filename+"'", ex);
++        }
++        catch (IOException ex) {
++            throw new RuntimeIOException("Failed to open '"+filename+"'", ex); 
++        }
++    }
++
++    /** Lock the file or throw {@link DBOpEnvException} */ 
++    public void lockEx() {
++        lockOperation(NoLockAction.EXCEPTION);
++    }
++    
++    /** Lock the file or wait. */ 
++    public void lockWait() {
++        lockOperation(NoLockAction.EXCEPTION);
++    }
++    
++    /** Lock a file, return true on success else false. */
++    public boolean tryLock() {
++        return lockOperation(NoLockAction.RETURN);
++    }
++
++    /** Release the lock - this must be paired with a "lock" operation. */
++    public void unlock() {
++        synchronized(sync) {
++            try {
++                fileLock.release();
++            } catch (IOException ex) { throw new RuntimeIOException("Failed to unlock '"+filepath+"'", ex); }
++        }
++    }
++
++    /** Return true if this process holds the lock. */
++    public boolean isLockedHere() {
++        if ( fileLock == null )
++            return false;
++        return fileLock.isValid();
++    }
++    
++    public Path getPath() {
++        return filepath;
++    }
++    
++    /** Take the lock.
++     * <p>
++     * Write our PID into the file and return true if it succeeds.
++     * <p>
++     * Try to get the existing PID if it fails.
++     */
++    private boolean lockOperation(NoLockAction action) {
++        synchronized(sync) {
++            if ( fileLock != null )
++                throw new AlreadyLocked("Failed to get a lock: file='"+filepath+"': Lock already held");
++            
++            try {
++                fileLock = (action == NoLockAction.WAIT) ? fileChannel.lock() : fileChannel.tryLock();
++                if ( fileLock == null ) {
++                    if ( action == NoLockAction.RETURN ) 
++                        return false ;
++                    // Read without the lock.
++                    // This isn't perfect but it is only providing helpful information.
++                    int pid = readProcessId(-99);
++                    if ( pid >= 0 )
++                        throw new DBOpEnvException("Failed to get a lock: file='"+filepath+"': held by process "+pid);
++                    throw new DBOpEnvException("Failed to get a lock: file='"+filepath+"': failed to get the holder's process id");
++                }
++                // Got the lock. Record our process id.
++                int pid = ProcessUtils.getPid(-1);
++                writeProcessId(pid);
++                return true;
++            } catch (IOException ex) {
++                if ( action == NoLockAction.RETURN ) 
++                    return false;
++                throw new DBOpEnvException("Failed to get a lock: file='"+filepath+"'", ex);
++            }
++        }
++    }
++    
++    // I/O for a process id. 
++    /** Read the file to get a process id */
++    private int readProcessId(int dft) throws IOException {
++        ByteBuffer bb = ByteBuffer.allocate(128);
++        fileChannel.position(0);
++        int len = fileChannel.read(bb);
++        fileChannel.position(0);
++        if ( len == 0 )
++            return dft;
++        if ( len == 128 )
++            // Too much.
++            return dft;
++        // Bug in Jena 3.3.0
++        //byte[] b = ByteBufferLib.bb2array(bb, 0, len);
++        bb.flip();
++        byte[] b = new byte[len];
++        bb.get(b);
++        
++        String pidStr = StrUtils.fromUTF8bytes(b);
++        // Remove all leadign and trailing (vertical and horizontal) whitespace.
++        pidStr = pidStr.replaceAll("[\\s\\t\\n\\r]+$", "");
++        pidStr = pidStr.replaceAll("^[\\s\\t\\n\\r]+", "");
++        try {
++            // Process id.
++            return Integer.parseInt(pidStr);
++        } catch (NumberFormatException ex) {
++            ex.printStackTrace();
++            Log.warn(this, "Bad process id: file='"+filepath+"': read='"+pidStr+"'");
++            return dft; 
++        }
++    }
++
++    /** Write the process id to the file. */
++    private void writeProcessId(int pid) throws IOException {
++        byte b[] = StrUtils.asUTF8bytes(Integer.toString(pid)+"\n");
++        fileChannel.truncate(0);
++        int len = fileChannel.write(ByteBuffer.wrap(b));
++        fileChannel.position(0);
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/SegmentedMemBuffer.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/SegmentedMemBuffer.java
index 0000000,0000000..abe06fd
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/file/SegmentedMemBuffer.java
@@@ -1,0 -1,0 +1,279 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.file;
++
++import java.nio.ByteBuffer ;
++import java.util.ArrayList ;
++import java.util.List ;
++
++import org.apache.jena.atlas.io.IO ;
++import org.apache.jena.atlas.lib.ByteBufferLib ;
++import org.apache.jena.atlas.logging.FmtLog ;
++import org.slf4j.Logger ;
++import org.slf4j.LoggerFactory ;
++
++/** A segmented, expanding buffer of bytes.
++ *  This class does not copy the underlying bytes when the file grows.
++ *  Hence, the performance is more predictable.
++ *  (Resizing a fixed size buffer is a copy c.f. performance issues with
++ *  {@code ArrayList} or {@code ByteArrayOutputStream}
++ *  as they go from small to large. 
++ */
++public class SegmentedMemBuffer {
++    private static Logger log = LoggerFactory.getLogger(SegmentedMemBuffer.class) ;
++    
++    // See java.nio.channels.FileChannel
++    private static int DFT_CHUNK = 1024*1024 ;
++    private static int DFT_SEGMENTS = 1000 ;
++    private final int CHUNK ;
++    // The file pointer and marker of the allocated end point.
++    private long dataLength = 0 ;
++    private List<byte[]> space = null ;
++    private boolean isOpen ;
++    private final boolean TRACKING = false ;
++
++    public SegmentedMemBuffer() { this(DFT_CHUNK) ; }
++    
++    public SegmentedMemBuffer(int chunk) { 
++        this.CHUNK = chunk ;
++        space = new ArrayList<>(DFT_SEGMENTS) ;
++        // "Auto open"
++        isOpen = true ;
++    }
++
++    public void open() {
++        isOpen = true ;
++//        if ( storage != null )
++//            throw new RuntimeIOException("Already open") ;
++//        storage = new ArrayList<>(DFT_SEGMENTS) ;
++    }
++    
++    public boolean isOpen() {
++        return space != null ;
++    }
++    
++    public int read(long posn, ByteBuffer bb) {
++        if ( TRACKING )
++            log("read<<%s", ByteBufferLib.details(bb));
++        checkOpen() ;
++        checkPosition(posn) ;
++        int len = bb.remaining() ;
++        if ( posn+dataLength > dataLength )
++            len = (int)(dataLength-posn) ;
++        arrayCopyOut(space, posn, bb);
++        return len ;
++    }
++
++    public int read(long posn, byte[] b) {
++        if ( TRACKING )
++            log("read<<[%d]", b.length) ;
++        return read$(posn, b, 0, b.length) ; 
++    }
++    
++    public int read(long posn, byte[] b, int start, int length) {
++        if ( TRACKING )
++            log("read<<[%d],%s,%s", b.length, start, length) ;
++        return read$(posn, b, start, length) ;
++    }
++    
++    private int read$(long posn, byte[] b, int start, int length) {
++        checkOpen() ;
++        if ( posn >= dataLength ) 
++            return -1 ;
++        checkPosition(posn) ;
++        if ( length == 0 )
++            return 0 ;
++        checkByteArray(b, start, length) ;
++        int len = length ;
++        if ( posn+length > dataLength )
++            len = (int)(dataLength-posn) ;
++        arrayCopyOut(space, posn, b, start, len) ;
++        return len ;
++    }
++
++    public void write(long posn, ByteBuffer bb ) {
++        if ( TRACKING )
++            log("read<<%s", ByteBufferLib.details(bb));
++        checkOpen() ;
++        if ( posn != dataLength )
++            checkPosition(posn) ;
++        arrayCopyIn(bb, space, posn) ;
++    }
++
++    public void write(long posn, byte[] b ) {
++        if ( TRACKING )
++            log("read<<[%d]", b.length) ;
++        write$(posn, b, 0, b.length) ; 
++    }
++
++    public void write(long posn, byte[] b, int start, int length) {
++        if ( TRACKING )
++            log("read<<[%d],%d,%d", b.length, start, length) ;
++        write$(posn, b, start, length);
++    }
++    
++    private void write$(long posn, byte[] b, int start, int length) {
++        checkOpen() ;
++        checkPosition(posn) ;
++        if ( length == 0 )
++            return ;
++        checkByteArray(b,start,length) ;
++        arrayCopyIn(b, start, space, dataLength, length) ;
++    }
++
++    public void truncate(long length) {
++        if ( TRACKING )
++            log("truncate(%d)", length) ;
++        if ( length < 0 )
++            IO.exception(String.format("truncate: bad length : %d", length)) ;
++        checkOpen() ;
++        dataLength = Math.min(dataLength, length) ;
++        // clear above?
++    }
++
++    public void sync() {
++        checkOpen() ;
++    }
++
++    public void close() {
++        if ( ! isOpen() )
++            return ;
++        isOpen = false ;
++        space.clear() ;
++        space = null ;
++    }
++
++    public long length() {
++        return dataLength ;
++    }
++
++    private void checkOpen() {
++        if ( ! isOpen )
++            IO.exception("Not open") ;
++    }
++
++    private void checkPosition(long posn) {
++        // Allows posn to be exactly the byte beyond the end  
++        if ( posn < 0 || posn > dataLength )
++            IO.exception(String.format("Position out of bounds: %d in [0,%d]", posn, dataLength)) ;
++    }
++
++    private void checkByteArray(byte[] b, int start, int length) {
++        if ( start < 0 || start >= b.length )
++            IO.exception(String.format("Start point out of bounds of byte array: %d in [0,%d)", start, b.length)) ;
++        if ( length < 0 || start+length > b.length )
++            IO.exception(String.format("Start/length out of bounds of byte array: %d/%d in [0,%d)", start, length, b.length)) ;
++    }
++    
++    private int getSegment(long posn) { return (int)(posn / CHUNK) ; }
++
++    private int getOffset(long posn) { return (int) (posn % CHUNK) ; }
++
++    // c.f. System.arrayCopy 
++    private void arrayCopyIn(byte[] src, final int srcStart, List<byte[]> dest, final long destStart, final int length) {
++        if ( length == 0 )
++            return ;
++        int len = length ;
++        int srcPosn = srcStart ;
++        int seg = getSegment(destStart) ;
++        int offset = getOffset(destStart) ;
++        while( len > 0 ) {
++            int z = Math.min(len, CHUNK-offset) ;
++            // Beyond end of file?
++            if ( seg >= dest.size() ) {
++                byte[] buffer = new byte[CHUNK] ;
++                space.add(buffer) ;
++            }
++            System.arraycopy(src, srcPosn, dest.get(seg), offset, z);
++            srcPosn += z ;
++            len -= z ;
++            seg += 1 ;
++            offset += z ;
++            offset %= CHUNK ;
++        }
++        dataLength = Math.max(dataLength, destStart+length) ;
++    }
++
++    private void arrayCopyOut(List<byte[]> src, final long srcStart, byte[] dest, final int destStart, final int length) {
++        int len = length ;
++        len = Math.min(len, (int)(dataLength-srcStart)) ;
++        int dstPosn = destStart ;
++        int seg = getSegment(srcStart) ;
++        int offset = getOffset(srcStart) ;
++        while( len > 0 ) {
++            int z = Math.min(len, CHUNK-offset) ;
++            System.arraycopy(src.get(seg), offset, dest, dstPosn, z);
++            dstPosn += z ;
++            len -= z ;
++            seg += 1 ;
++            offset += z ;
++            offset %= CHUNK ;
++        }
++    }
++
++    private void arrayCopyIn(ByteBuffer bb, List<byte[]> dest, long destStart) {
++        if ( bb.remaining() == 0 )
++            return ;
++        int length = bb.remaining() ;
++        int len = bb.remaining() ;
++        int srcPosn = bb.position() ;
++        int seg = getSegment(destStart) ;
++        int offset = getOffset(destStart) ;
++        while( len > 0 ) {
++            int z = Math.min(len, CHUNK-offset) ;
++            // Beyond end of file?
++            if ( seg >= dest.size() ) {
++                byte[] buffer = new byte[CHUNK] ;
++                space.add(buffer) ;
++            }
++            byte[] bytes = dest.get(seg) ;
++            bb.get(bytes, offset, z) ;
++            srcPosn += z ;
++            len -= z ;
++            seg += 1 ;
++            offset += z ;
++            offset %= CHUNK ;
++        }
++        dataLength = Math.max(dataLength, destStart+length) ;
++    }
++
++    private void arrayCopyOut(List<byte[]> src, long srcStart, ByteBuffer bb) {
++        int len = bb.remaining() ;
++        len = Math.min(len, (int)(dataLength-srcStart)) ;
++        int dstPosn = bb.position() ;
++        int seg = getSegment(srcStart) ;
++        int offset = getOffset(srcStart) ;
++        while( len > 0 ) {
++            int z = Math.min(len, CHUNK-offset) ;
++            byte[] bytes = src.get(seg) ;
++            bb.put(bytes, offset, z) ;
++            dstPosn += z ;
++            len -= z ;
++            seg += 1 ;
++            offset += z ;
++            offset %= CHUNK ;
++        }
++    }
++    
++    private void log(String fmt, Object... args) {
++        if ( TRACKING )
++            FmtLog.debug(log, fmt, args); 
++    }
++
++}
++

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/BlockConverter.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/BlockConverter.java
index 0000000,0000000..d77165c
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/BlockConverter.java
@@@ -1,0 -1,0 +1,34 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.page;
++
++import org.seaborne.dboe.base.block.Block ;
++import org.seaborne.dboe.base.block.BlockType ;
++
++/** Convert between Blocks and typed Pages */ 
++public interface BlockConverter<T extends Page>
++{
++    /** Create a T, given an existing Block */
++    public T fromBlock(Block block) ;
++    
++    /** Make a block, given a T */
++    public Block toBlock(T t) ;
++    
++    /** Create a new T from an uninitialized Block */ 
++    public T createFromBlock(Block block, BlockType bType) ;
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/Page.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/Page.java
index 0000000,0000000..02dca0b
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/Page.java
@@@ -1,0 -1,0 +1,44 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.page;
++
++import org.apache.jena.atlas.io.Printable ;
++import org.seaborne.dboe.base.block.Block ;
++
++public interface Page extends Printable
++{
++    public static final int NO_ID   = -1 ;
++    
++    /** Pages are addressed ints (a page ref does in on-disk blocks)
++     * although block are address in longs
++     */  
++    public int getId() ;
++    
++    /** Return a string for display that identifies this Page */
++    public String getRefStr() ;
++    
++    /** Return the block associated with this page */ 
++    public Block getBackingBlock() ;
++    
++    /**
++     * The underlying block for this page has changed (e.g. it's been
++     * promoted and the promotion may have caused the block to change.
++     */
++    public abstract void reset(Block block) ;
++
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBase.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBase.java
index 0000000,0000000..ff1f5db
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBase.java
@@@ -1,0 -1,0 +1,57 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.page;
++
++import org.seaborne.dboe.DBOpEnvException ;
++import org.seaborne.dboe.base.block.Block ;
++
++/** A page with a byte buffer */
++public abstract class PageBase implements Page
++{
++    private int id ;
++    private Block block ;
++
++    protected PageBase(Block block)
++    {
++        this.block = block ;
++        long x = block.getId() ;
++        if ( x < 0 )
++            throw new DBOpEnvException("Page id is negative: "+x) ;
++        if ( x > Integer.MAX_VALUE )
++            throw new DBOpEnvException("Page id is large than MAX_INT: "+x) ;
++        this.id = block.getId().intValue() ;
++    }
++    
++    @Override
++    final public void reset(Block block2)
++    { 
++//        if ( block2.getId() != id )
++//            Log.warn(this, "Block id changed: "+id+" => "+block2.getId()) ;
++        id = block2.getId().intValue() ;
++        _reset(block2) ; 
++        this.block = block2 ;
++    } 
++
++    protected abstract void _reset(Block block) ;
++
++    @Override
++    final public Block getBackingBlock()    { return block ; }
++
++    @Override
++    final public int getId()                { return id ; }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBlockMgr.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBlockMgr.java
index 0000000,0000000..533d6d9
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/page/PageBlockMgr.java
@@@ -1,0 -1,0 +1,212 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.page;
++
++import org.apache.jena.atlas.lib.Closeable ;
++import org.apache.jena.atlas.lib.InternalErrorException ;
++import org.apache.jena.atlas.logging.Log ;
++import org.seaborne.dboe.base.block.Block ;
++import org.seaborne.dboe.base.block.BlockMgr ;
++import org.seaborne.dboe.base.block.BlockType ;
++
++/** Engine that wraps from blocks to typed pages. */
++
++public class PageBlockMgr<T extends Page> implements Closeable
++{
++    protected final BlockMgr blockMgr ;
++    protected BlockConverter<T> pageFactory ;
++
++    protected PageBlockMgr(BlockConverter<T> pageFactory, BlockMgr blockMgr) { 
++        this.pageFactory = pageFactory ;
++        this.blockMgr = blockMgr ;
++    }
++   
++    // Sometimes, the subclass must pass null to the constructor then call this. 
++    protected void setConverter(BlockConverter<T> pageFactory) { this.pageFactory = pageFactory ; }
++    
++    public BlockMgr getBlockMgr()   { return blockMgr ; } 
++    
++    public long allocLimit()        { return blockMgr.allocLimit() ; }
++
++    public void resetAlloc(long boundary) {
++        blockMgr.resetAlloc(boundary) ;
++    }
++
++    /** Allocate a new thing */
++    public T create(BlockType bType) {
++        Block block = blockMgr.allocate(-1) ;
++        block.setModified(true) ;
++        T page = pageFactory.createFromBlock(block, bType) ;
++        return page ;
++    }
++    
++    /**
++     * Fetch a block for reading.
++     * @param id Block to fetch
++     */
++    public T getRead(int id) {
++        return getRead$(id) ;
++    }
++
++    /**
++     * Fetch a block for reading.
++     * 
++     * @param id    Block to fetch
++     * @param referencingId
++     *            Id of block referring to this one. 
++     *            For example, a parent in a tree.
++     *            May be negative for "none" or "meaningless".
++     */
++    public T getRead(int id, int referencingId) {
++        return getRead$(id) ;
++    }
++    
++    /**
++     * Fetch a block for writing.
++     * @param id Block to fetch
++     */
++    public T getWrite(int id) {
++        return getWrite$(id) ;
++    }
++    
++    /**
++     * Fetch a block for writing.
++     * 
++     * @param id  Block to fetch
++     * @param referencingId
++     *            Id of block referring to this one. 
++     *            For example, a parent in a tree.
++     *            May be -1 for "none" or "meaningless".
++     */
++    public T getWrite(int id, int referencingId) {
++        return getWrite$(id) ;
++    }
++
++    // ---- The read and write worker operations.
++    
++    final protected T getRead$(int id) { 
++        Block block = blockMgr.getRead(id) ;
++        // Blocks from the BlockMgrCache may be write-dirty so this test
++        // is wrong in that situation.  It is better to use the block as-is
++        // otherwise we'd have two blocks of a given id with different flags
++        // that confuse the block caching.  
++//        if ( block.isModified() ) {
++//            System.err.println("getRead - isModified - "+blockMgr.getLabel()+"["+id+"]") ;
++////            block = new Block(block.getId(), block.getByteBuffer());
++////            block.setModified(false); 
++//            // Debug.
++//            //blockMgr.getRead(id) ;
++//        }
++        T page = pageFactory.fromBlock(block) ;
++        return page ;
++    }
++    
++    final protected T getWrite$(int id) {
++        Block block = blockMgr.getWrite(id) ;
++        block.setReadOnly(false) ;
++        T page = pageFactory.fromBlock(block) ;
++        return page ;
++    }
++
++    // ---- 
++    
++    public void put(T page) {
++        write(page) ;
++        release(page) ;
++    }
++
++    public void write(T page) {
++        Block blk = pageFactory.toBlock(page) ;
++        blockMgr.write(blk) ;
++    }
++
++    public void release(Page page) {
++        Block block = page.getBackingBlock() ;
++        blockMgr.release(block) ;
++    }
++
++    private void warn(String string) {
++        Log.warn(this, string) ;
++    }
++
++    public void free(Page page) {
++        Block block = page.getBackingBlock() ;
++        blockMgr.free(block) ;
++    }
++
++    
++    /** Promote a page to be writable in-place (block id does not change, hnce page does not change id). */
++    public void promoteInPlace(Page page) {
++        Block block = page.getBackingBlock() ;
++        block.getByteBuffer().rewind() ;
++        Block block2 = blockMgr.promote(block) ; 
++        block2.setReadOnly(false) ;
++        if ( block2.getId() != block.getId() )
++            throw new InternalErrorException("Block id changed") ;
++        if ( block2 == block )
++            return ;
++        // Change - reset Block in page.
++        // The details should not have changed.
++        // page.reset(block2) ;
++    }
++    
++    /** Promote a page - return 'true' if the block changed (.reset()) will have been called */ 
++    public boolean promoteDuplicate(Page page) {
++        Block block = page.getBackingBlock() ;
++        block.getByteBuffer().rewind() ;
++        
++        // --- TODO Always new
++        Block block2 =  blockMgr.allocate(-1) ;
++        block2.getByteBuffer().put(block.getByteBuffer()) ;
++        block2.getByteBuffer().rewind() ;
++        block2.setReadOnly(false) ;
++
++        if ( block2 == block )
++            return false ;
++        // Change - reset Block in page.
++        page.reset(block2) ;
++        return true ;
++    }
++
++    public boolean valid(int id) {
++        return blockMgr.valid(id) ;
++    }
++
++    public void dump() {
++        for ( int idx = 0 ; valid(idx) ; idx++ ) {
++            T page = getRead(idx, -1) ;
++            System.out.println(page) ;
++            release(page) ;
++        }
++    }
++
++    /** Signal the start of an update operation */
++    public void startUpdate()       { blockMgr.beginUpdate() ; }
++    
++    /** Signal the completion of an update operation */
++    public void finishUpdate()      { blockMgr.endUpdate() ; }
++
++    /** Signal the start of an update operation */
++    public void startRead()         { blockMgr.beginRead() ; }
++    
++    /** Signal the completeion of an update operation */
++    public void finishRead()        { blockMgr.endRead() ; }
++
++    @Override
++    public void close()             { blockMgr.close(); }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
index 0000000,0000000..869e705
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/Record.java
@@@ -1,0 -1,0 +1,188 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.record;
++
++import static java.lang.String.format ;
++
++import java.util.Arrays ;
++
++import org.apache.jena.atlas.lib.Bytes ;
++import org.seaborne.dboe.sys.SystemIndex ;
++
++/** A record is pair of key and value.  It may be all key, in which case value is null. */
++
++final
++public class Record //implements Comparable<Record>
++{
++    /*
++     * Records of fixed size (controlled by the factory).
++     */
++    public static final Record NO_REC = null ;
++    
++    final private byte[] key ;
++    final private byte[] value ;
++    
++    public Record(byte[] key, byte[] value)
++    { 
++        this.key = key ;
++        this.value = value ;
++        if ( SystemIndex.Checking )
++        {
++            if ( value != null && value.length == 0 )
++                throw new RecordException("Zero length value") ;
++        }
++    }
++    
++    public byte[] getKey()          { return key ; }
++    public byte[] getValue()        { return value ; }
++
++    public boolean hasSeparateValue() { return value!=null ; }
++    
++    @Override
++    public int hashCode()
++    { 
++        return Arrays.hashCode(key) ^ Arrays.hashCode(value) ;
++    } 
++
++    @Override
++    public boolean equals(Object other)
++    {
++        if ( this == other ) return true ;
++        if ( ! ( other instanceof Record ) ) return false ;
++        Record r = (Record)other ;
++        return compareByKeyValue(this, r) == 0 ;
++    }
++    
++    @Override
++    public String toString()
++    {
++        if ( value == null )
++            return str(key) ;
++        return str(key)+":"+str(value) ;
++    }
++    
++    /** Is the key of record1 == key of record2 */
++    public static boolean keyEQ(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x == 0 ;
++    }
++
++    /** Is the key of record1 != key of record2 */
++    public static boolean keyNE(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x != 0 ;
++    }
++
++    /** Is the key of record1 < key of record2 */
++    public static boolean keyLT(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x < 0 ;
++    }
++
++    /** Is the key of record1 <= key of record2 */
++    public static boolean keyLE(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x <= 0 ;
++    }
++    
++    /** Is the key of record1 >= key of record2 */
++    public static boolean keyGE(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x >= 0 ;
++    }
++    
++    /** Is the key of record1 > key of record2 */
++    public static boolean keyGT(Record record1, Record record2)
++    {
++        int x = compareByKey(record1, record2) ;
++        return x > 0 ;
++    }
++    
++    /** Is (key, value) of record1 equal to (key,value) of record2 */
++    public static boolean equals(Record record1, Record record2)
++    {
++        int x = compareByKeyValue(record1, record2) ;
++        return x == 0 ;
++    }
++
++
++    
++    static public String str(byte[] b)
++    {
++        StringBuilder str = new StringBuilder() ;
++        for ( byte aB : b )
++        {
++            str.append( format( "%02x", aB ) );
++        }
++        return str.toString() ;
++    }
++    
++    public static int compareByKey(Record record1, Record record2)
++    {
++        checkKeyCompatible(record1, record2) ;
++        return Bytes.compare(record1.key, record2.key) ; 
++    }
++    
++    public static int compareByKeyValue(Record record1, Record record2)
++    {
++        checkCompatible(record1, record2) ;
++        int x = Bytes.compare(record1.key, record2.key) ;
++        if ( x == 0 )
++        {
++            if ( record1.value != null )
++                x = Bytes.compare(record1.value, record2.value) ;
++        }
++        return x ;
++    }
++
++    static void checkCompatible(Record record1, Record record2)
++    {
++        if ( ! compatible(record1, record2, true) )
++            throw new RecordException(format("Incompatible: %s, %s", record1, record2)) ;
++    }
++    
++    static void checkKeyCompatible(Record record1, Record record2)
++    {
++        if ( ! compatible(record1, record2, false) )
++            throw new RecordException(format("Incompatible: %s, %s", record1, record2)) ;
++    }
++    
++    static boolean compatible(Record record1, Record record2, boolean checkValue)
++    {
++        if ( record1.key.length != record2.key.length )
++            return false ;
++        
++        if ( checkValue )
++        {
++            if ( record1.value == null && record2.value == null )
++                return true ;
++            if ( record1.value == null )
++                return false ;
++            if ( record2.value == null )
++                return false ;
++            if ( record1.value.length != record2.value.length )
++                return false;
++        }
++        return true ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
index 0000000,0000000..7c192c8
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordException.java
@@@ -1,0 -1,0 +1,26 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.record;
++
++import org.seaborne.dboe.DBOpEnvException ;
++
++public class RecordException extends DBOpEnvException
++{
++    public RecordException() {}
++    public RecordException(String msg) { super(msg) ; }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
index 0000000,0000000..1413312
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordFactory.java
@@@ -1,0 -1,0 +1,183 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.record;
++
++import static java.lang.String.format;
++
++import java.nio.ByteBuffer;
++
++/** Record creator */
++final
++public class RecordFactory
++{
++    private final int keyLength;
++    private final int valueLength;
++    private final int slotLen;
++    private final boolean checking = false;
++
++    public RecordFactory(int keyLength, int valueLength) {
++        if ( keyLength <= 0 )
++            throw new IllegalArgumentException("Bad key length: " + keyLength);
++        if ( valueLength < 0 )
++            throw new IllegalArgumentException("Bad value length: " + valueLength);
++
++        this.keyLength = keyLength;
++        this.valueLength = valueLength;
++        this.slotLen = keyLength + valueLength;
++    }
++
++    /** Return a RecordFactory that makes key-only records of the same key size */
++    public RecordFactory keyFactory() {
++        return new RecordFactory(keyLength, 0);
++    }
++
++    /** Create a key-only record, allocating blank space for the key */
++    public Record createKeyOnly() {
++        return create(new byte[keyLength], null);
++    }
++
++    /** Create a key-only record */
++    public Record createKeyOnly(Record record) {
++        checkKey(record.getKey());
++        if ( record.getValue() == null )
++            return record;
++
++        return create(record.getKey(), null);
++    }
++
++    /** Create a key and value record (value uninitialized) */
++    public Record create(byte[] key) {
++        checkKey(key);
++        byte[] v = null;
++        if ( valueLength > 0 )
++            v = new byte[valueLength];
++        return create(key, v);
++    }
++
++    /** Create a record, allocating space for the key and value (if any) */
++    public Record create() {
++        return create(new byte[keyLength], (valueLength > 0) ? new byte[valueLength] : null);
++    }
++
++    /** Create a key and value record */
++    public Record create(byte[] key, byte[] value) {
++        check(key, value);
++        return new Record(key, value);
++    }
++
++    public void insertInto(Record record, ByteBuffer bb, int idx) {
++        check(record);
++        bb.position(idx * slotLen);
++        bb.put(record.getKey(), 0, keyLength);
++        if ( hasValue() && record.getValue() != null )
++            bb.put(record.getValue(), 0, valueLength);
++    }
++
++    public static final RecordMapper<Record> mapperRecord = new RecordMapper<Record>() {
++        /*
++         * RecordRangeIterator calls getRecordBuffer().get(slotidx);
++         */
++        @Override
++        public Record map(ByteBuffer bb, int idx, byte[] keyBytes, RecordFactory factory) {
++            byte[] key = new byte[factory.keyLength];
++            byte[] value = (factory.hasValue() ? new byte[factory.valueLength] :null );
++            // Is it better to avoid the synchronize needed for bb.position./bb.get
++            // but not use a (relative) bulk bb.get(byte[],,) which may be a native operation?
++            
++//            int posnKey = idx*slotLen;
++//            // Avoid using position() so we can avoid needing synchronized.
++//            copyInto(key, bb, posnKey, keyLength);
++//            if ( value != null )
++//            {
++//                int posnValue = idx*slotLen+keyLength;
++//                copyInto(value, bb, posnValue, valueLength);
++//            }
++            
++            // Using bb.get(byte[],,) may be potentially faster but requires the synchronized
++            // There's no absolute version.
++            synchronized(bb)
++            {
++                bb.position(idx*factory.slotLen);
++                bb.get(key, 0, factory.keyLength);
++                if ( value != null )
++                    bb.get(value, 0, factory.valueLength);
++            }
++            if ( keyBytes != null )
++                System.arraycopy(key, 0, keyBytes, 0, factory.keyLength);
++            return factory.create(key, value);
++        }
++    };
++    
++    public <X> X access(ByteBuffer bb, int idx, byte[] keyBytes, RecordMapper<X> mapper) {
++        return mapper.map(bb, idx, keyBytes, this); 
++    }
++    
++    public Record buildFrom(ByteBuffer bb, int idx) {
++        return access(bb, idx, null, mapperRecord); 
++    }
++    
++    private final void copyInto(byte[] dst, ByteBuffer src, int start, int length) {
++        // Thread safe.
++        for ( int i = 0; i < length; i++ )
++            dst[i] = src.get(start+i);
++        // Would otherwise be ...
++//        src.position(start);
++//        src.get(dst, 0, length);
++    }
++    
++    public boolean hasValue()   { return valueLength > 0; }
++
++    public int recordLength()   { return keyLength + valueLength; }
++    
++    public int keyLength()      { return keyLength; }
++
++    public int valueLength()    { return valueLength; }
++    
++    @Override
++    public String toString() {
++        return format("<RecordFactory k=%d v=%d>", keyLength, valueLength);
++    }
++
++    private final void check(Record record) {
++        if ( ! checking ) return;
++        check(record.getKey(), record.getValue());
++    }
++
++    private final void checkKey(byte[] k) {
++        if ( ! checking ) return;
++        if ( k == null )
++            throw new RecordException("Null key byte[]");
++        if ( keyLength != k.length )
++            throw new RecordException(format("Key length error: This RecordFactory manages records of key length %d, not %d", keyLength,
++                                             k.length));
++    }
++
++    private final void check(byte[] k, byte[] v) {
++        if ( ! checking ) return;
++        checkKey(k);
++        if ( valueLength <= 0 ) {
++            if ( v != null )
++                throw new RecordException("Value array error: This RecordFactory manages records that are all key");
++        } else {
++            // v == null for a key-only record from this factory.
++            if ( v != null && v.length != valueLength )
++                throw new RecordException(format("This RecordFactory manages record of value length %d, not (%d,-)", valueLength,
++                                                 v.length));
++        }
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
index 0000000,0000000..34968e2
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/record/RecordMapper.java
@@@ -1,0 -1,0 +1,30 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.record;
++
++import java.nio.ByteBuffer ;
++
++/** Allow bytes to pull directly out of storage with no copy to record;
++ *  also extract the key bytes into an array.
++ *  This operation MUST not retain any references to the ByteBuffer storage space.
++ *  All in order to avoid remapping Records to higher level objects,
++ *  which churns objects (GC issue) and avoids a copy.
++ */
++public interface RecordMapper<X> {
++    X map(ByteBuffer bb, int entryIdx, byte key[], RecordFactory recFactory) ;
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
index 0000000,0000000..84b251d
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPage.java
@@@ -1,0 -1,0 +1,100 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.recordbuffer;
++
++import org.seaborne.dboe.base.block.Block ;
++import org.seaborne.dboe.base.page.Page ;
++import org.seaborne.dboe.base.record.RecordFactory ;
++import org.seaborne.dboe.sys.Sys ;
++
++/**
++ * B+Tree records nodes and hash buckets.
++ * Add link field to a RecordBufferPageBase
++ */
++
++public final class RecordBufferPage extends RecordBufferPageBase
++{
++    // Why not straight to BPlusTreeRecords?
++    // 1 - this may be useful in its own right as a sequence of records on-disk.
++    // 2 - BPlusTreeRecords inherits from BPlusTreePage
++    
++    
++    // To Constants
++    // Offsets
++    //    final public static int COUNT      = 0 ;
++    // Adds this field over RecordBufferPageBase
++    final public static int LINK            = 4 ;
++    final private static int FIELD_LENGTH   = Sys.SizeOfInt ; // Length of the space needed here (not count)
++
++    private int link = Page.NO_ID ;
++    
++    public final int getLink() { return link ; }
++    
++    public void setLink(int link)
++    { 
++        this.link = link ;
++        getBackingBlock().getByteBuffer().putInt(LINK, link) ;
++    }
++    
++    @Override
++    protected void _reset(Block block)
++    { 
++        // TODO -- should this be format?
++        // Print this 
++        super.rebuild(block, this.getCount()) ;
++        //?? use .format
++//        // TODO WRONG : block is overlying.
++//        this.link = block.getByteBuffer().getInt(LINK) ;
++    }
++
++    public static int calcRecordSize(RecordFactory factory, int blkSize)
++    { return RecordBufferPageBase.calcRecordSize(factory, blkSize, FIELD_LENGTH) ; }
++    
++    public static int calcBlockSize(RecordFactory factory, int maxRec)
++    { return RecordBufferPageBase.calcBlockSize(factory, maxRec, FIELD_LENGTH) ; }
++    
++    /** The construction methods */
++    public static RecordBufferPage createBlank(Block block,RecordFactory factory)
++    {
++        int count = 0 ;
++        int linkId = NO_ID ;
++        return new RecordBufferPage(block, factory, count, linkId) ;
++    }
++
++    public static RecordBufferPage format(Block block, RecordFactory factory)
++    {
++        int count = block.getByteBuffer().getInt(COUNT) ;
++        int linkId = block.getByteBuffer().getInt(LINK) ;
++        return new RecordBufferPage(block, factory, count, linkId) ;
++    } 
++    
++    private RecordBufferPage(Block block, RecordFactory factory, int count, int linkId)  
++    {
++        super(block, FIELD_LENGTH, factory, count) ;
++        this.link = linkId ;
++    }
++    
++    @Override
++    public String toString()
++    { return String.format("RecordBufferPage[id=%d,link=%d]: %s", getBackingBlock().getId(), getLink(), recBuff) ; }
++
++    @Override
++    public String getRefStr() {
++        return String.format("RecordBufferPage[id=%d]", getBackingBlock().getId()) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
index 0000000,0000000..9559008
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageBase.java
@@@ -1,0 -1,0 +1,105 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.recordbuffer ;
++
++import java.nio.ByteBuffer ;
++
++import org.apache.jena.atlas.io.IndentedWriter ;
++import org.seaborne.dboe.base.block.Block ;
++import org.seaborne.dboe.base.buffer.RecordBuffer ;
++import org.seaborne.dboe.base.page.PageBase ;
++import org.seaborne.dboe.base.record.RecordFactory ;
++import org.seaborne.dboe.sys.Sys ;
++
++/**
++ * The on-disk form of a block of a single RecordBuffer
++ * (i.e. this is not part of a BTree/BPlusTree branch node).
++ * This must be compatible with B+Tree records nodes and hashbuckets.
++ */
++
++public abstract class RecordBufferPageBase extends PageBase // implements Page
++{
++    // Field offsets
++    final public static int     COUNT        = 0 ;
++    // Length due to this class - subclasses may use more overhead.
++    final private static int    FIELD_LENGTH = Sys.SizeOfInt ;
++
++    protected final int         headerLength ;
++
++    // Interface: "Page" - id, byteBuffer, count
++    protected RecordBuffer      recBuff ;
++    private final RecordFactory factory ;
++
++    // private int offset ; // Bytes of overhead.
++
++    public static int calcRecordSize(RecordFactory factory, int blkSize, int headerOffset) {
++        // Length = X*recordLength + HEADER
++        int x = blkSize - totalOffset(headerOffset) ;
++        return x / factory.recordLength() ;
++    }
++
++    public static int calcBlockSize(RecordFactory factory, int maxRec, int headerOffset) {
++        return totalOffset(headerOffset) + factory.recordLength() * maxRec ;
++    }
++
++    private static int totalOffset(int headerOffset) {
++        return FIELD_LENGTH + headerOffset ;
++    }
++
++    protected RecordBufferPageBase(Block block, int offset, RecordFactory factory, int count) {
++        // This code knows the alignment of the records in the ByteBuffer.
++        super(block) ;
++        this.headerLength = FIELD_LENGTH + offset ; // NB +4 for the count field
++        this.factory = factory ;
++        rebuild(block, count) ;
++    }
++
++    protected void rebuild(Block block, int count) {
++        ByteBuffer bb = block.getByteBuffer() ;
++        bb.clear() ;
++        bb.position(headerLength) ;
++        bb = bb.slice() ;
++        this.recBuff = new RecordBuffer(bb, factory, count) ;
++    }
++
++    public final RecordBuffer getRecordBuffer() {
++        return recBuff ;
++    }
++
++    public final int getCount() {
++        return recBuff.size() ;
++    }
++
++    public final int getMaxSize() {
++        return recBuff.maxSize() ;
++    }
++
++    public void setCount(int count) {
++        recBuff.setSize(count) ;
++    }
++
++    @Override
++    public String toString() {
++        return String.format("RecordBufferPageBase[id=%d]: %s", getBackingBlock().getId(), recBuff) ;
++    }
++
++    @Override
++    public void output(IndentedWriter out) {
++        out.print(toString()) ;
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
index 0000000,0000000..09275a9
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordBufferPageMgr.java
@@@ -1,0 -1,0 +1,90 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.recordbuffer;
++
++import static org.seaborne.dboe.base.recordbuffer.RecordBufferPage.* ;
++
++import java.nio.ByteBuffer;
++
++import org.seaborne.dboe.base.block.Block ;
++import org.seaborne.dboe.base.block.BlockMgr ;
++import org.seaborne.dboe.base.block.BlockType ;
++import org.seaborne.dboe.base.page.BlockConverter ;
++import org.seaborne.dboe.base.page.PageBlockMgr ;
++import org.seaborne.dboe.base.record.RecordException ;
++import org.seaborne.dboe.base.record.RecordFactory ;
++
++/** Manager for a block that is all records.  
++ *  This must be compatible with B+Tree records nodes and with hash buckets. 
++ */
++
++final
++public class RecordBufferPageMgr extends PageBlockMgr<RecordBufferPage>
++{
++    private final RecordFactory factory ;
++    
++    public RecordBufferPageMgr(RecordFactory factory, BlockMgr blockMgr) {
++        super(new Block2RecordBufferPage(factory), blockMgr) ;
++        this.factory = factory ;
++    }
++
++    public RecordFactory getRecordFactory() { return factory ; }
++    
++    public RecordBufferPage create() {
++        return super.create(BlockType.RECORD_BLOCK) ;
++    }
++    
++    public static class Block2RecordBufferPage implements BlockConverter<RecordBufferPage> {
++        private RecordFactory factory ;
++
++        public Block2RecordBufferPage(RecordFactory factory) {
++            this.factory = factory ;
++        }
++
++        @Override
++        public RecordBufferPage createFromBlock(Block block, BlockType blkType) {
++            if ( blkType != BlockType.RECORD_BLOCK )
++                throw new RecordException("Not RECORD_BLOCK: " + blkType) ;
++            // Initially empty
++            RecordBufferPage rb = RecordBufferPage.createBlank(block, factory) ;
++            return rb ;
++        }
++
++        @Override
++        public RecordBufferPage fromBlock(Block block) {
++            synchronized (block) // [[TxTDB:TODO] needed? Right place?
++            {
++                RecordBufferPage rb = RecordBufferPage.format(block, factory) ;
++                // int count = block.getByteBuffer().getInt(COUNT) ;
++                // int linkId = block.getByteBuffer().getInt(LINK) ;
++                // RecordBufferPage rb = new RecordBufferPage(block, linkId,
++                // factory, count) ;
++                return rb ;
++            }
++        }
++
++        @Override
++        public Block toBlock(RecordBufferPage rbp) {
++            int count = rbp.getRecordBuffer().size() ;
++            ByteBuffer bb = rbp.getBackingBlock().getByteBuffer() ;
++            bb.putInt(COUNT, rbp.getCount()) ;
++            bb.putInt(LINK, rbp.getLink()) ;
++            return rbp.getBackingBlock() ;
++        }
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
index 0000000,0000000..92ae3c1
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/base/recordbuffer/RecordRangeIterator.java
@@@ -1,0 -1,0 +1,161 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.base.recordbuffer;
++
++import static org.apache.jena.atlas.lib.Alg.decodeIndex ;
++
++import java.util.Iterator ;
++import java.util.NoSuchElementException ;
++
++import org.apache.jena.atlas.lib.Bytes ;
++import org.apache.jena.atlas.lib.Closeable ;
++import org.seaborne.dboe.base.StorageException ;
++import org.seaborne.dboe.base.block.BlockException ;
++import org.seaborne.dboe.base.record.Record ;
++import org.seaborne.dboe.base.record.RecordMapper ;
++
++final public
++class RecordRangeIterator<X> implements Iterator<X>, Closeable
++{
++    /** Iterate over a range of fromRec (inclusive) to toRec (exclusive) */
++    public static <X> Iterator<X> iterator(int pageId, Record fromRec, Record toRec, RecordBufferPageMgr pageMgr, RecordMapper<X> mapper)
++    {
++        if ( ! pageMgr.valid(pageId) ) {
++            String msg = "RecordRangeIterator.iterator -- No such block (pageId="+pageId+", fromRec="+fromRec+", toRec="+toRec+ ")" ;
++//            System.err.println(msg) ;
++//            System.exit(0) ;
++            throw new BlockException(msg) ;
++        }
++        return new RecordRangeIterator<>(pageId, fromRec, toRec, pageMgr, mapper) ;
++    }
++
++    private final RecordMapper<X> mapper ;
++    
++    private RecordBufferPage currentPage ;      // Set null when finished.
++    
++    private int currentIdx ;
++    private final RecordBufferPageMgr pageMgr ;
++    private final Record maxRec ;
++    private final Record minRec ;
++    
++    private X slot = null ;
++    private final byte[] keySlot ;
++
++    private long countRecords = 0 ;
++    private long countBlocks = 0 ;
++
++    private RecordRangeIterator(int id, Record fromRec, Record toRec, RecordBufferPageMgr pageMgr, RecordMapper<X> mapper) {
++        currentIdx = 0 ;
++        this.pageMgr = pageMgr;
++        this.minRec = fromRec ;
++        this.maxRec = toRec ;
++        this.mapper = mapper ;
++        this.keySlot = new byte[pageMgr.getRecordFactory().keyLength()] ;
++        
++        if ( toRec != null && fromRec != null && Record.keyLE(toRec, fromRec) ) {
++            currentPage = null ;
++            return ;
++        }
++
++        pageMgr.startRead();
++        currentPage = pageMgr.getRead(id) ;
++        
++        if ( currentPage.getCount() == 0 ) {
++            // Empty page.
++            close() ;
++            return ;
++        }
++
++        if ( fromRec != null ) {
++            currentIdx = currentPage.getRecordBuffer().find(fromRec) ;
++            if ( currentIdx < 0 )
++                currentIdx = decodeIndex(currentIdx) ;
++        }        
++    }
++
++    @Override
++    public boolean hasNext() {
++        if ( slot != null )
++            return true ;
++        if ( currentPage == null )
++            return false ;
++        // Set slot.
++        while (currentIdx >= currentPage.getCount()) {
++            // Move to next.
++            int link = currentPage.getLink() ;
++            if ( link < 0 ) {
++                close() ;
++                return false ;
++            }
++
++            if ( currentPage != null )
++                pageMgr.release(currentPage) ;
++
++            RecordBufferPage nextPage = pageMgr.getRead(link) ;
++            // Check currentPage -> nextPage is strictly increasing keys.
++            if ( false ) {
++                Record r1 = currentPage.getRecordBuffer().getHigh() ;
++                Record r2 = nextPage.getRecordBuffer().getLow() ;
++                if ( Record.keyGE(r1, r2) )
++                    throw new StorageException("RecordRangeIterator: records not strictly increasing: " + r1 + " // " + r2) ;
++            }
++            currentPage = nextPage ;
++            countBlocks++ ;
++            currentIdx = 0 ;
++        }
++
++        slot = currentPage.getRecordBuffer().access(currentIdx, keySlot, mapper) ;
++        currentIdx++ ;
++
++        if ( maxRec != null && Bytes.compare(keySlot, maxRec.getKey()) >= 0 ) {
++            close() ;
++            return false ;
++        }
++
++        if ( slot == null ) {
++            close() ;
++            return false ;
++        }
++        countRecords++ ;
++        return true ;
++    }
++
++    @Override
++    public void close() {
++        if ( currentPage == null )
++            return ;
++        pageMgr.release(currentPage) ;
++        pageMgr.finishRead();
++        currentPage = null ;
++        currentIdx = -99 ;
++        slot = null ;
++    }
++
++    @Override
++    public X next() {
++        if ( !hasNext() )
++            throw new NoSuchElementException() ;
++        X x = slot ;
++        slot = null ;
++        return x ;
++    }
++
++    final public long getCountRecords()     { return countRecords ; }
++
++    final public long getCountBlocks()      { return countBlocks ; }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
index 0000000,0000000..5f7ea6a
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/jenax/Txn.java
@@@ -1,0 -1,0 +1,41 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.jenax;
++
++import java.util.function.Supplier ;
++
++import org.apache.jena.sparql.core.Transactional ;
++
++/** Indirection to allow for modification for TDB2 capabilities */
++public class Txn {
++    public static <T extends Transactional> void executeRead(T txn, Runnable r) {
++        org.apache.jena.system.Txn.executeRead(txn, r);
++    }
++
++    public static <T extends Transactional, X> X calculateRead(T txn, Supplier<X> r) {
++        return org.apache.jena.system.Txn.calculateRead(txn, r);
++    }
++
++    public static <T extends Transactional> void executeWrite(T txn, Runnable r) {
++        org.apache.jena.system.Txn.executeWrite(txn, r);
++    }
++
++    public static <T extends Transactional, X> X calculateWrite(T txn, Supplier<X> r) {
++        return org.apache.jena.system.Txn.calculateWrite(txn, r);
++    }
++}

http://git-wip-us.apache.org/repos/asf/jena/blob/d9da3592/jena-db/dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
----------------------------------------------------------------------
diff --cc jena-db/dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
index 0000000,0000000..e4e709f
new file mode 100644
--- /dev/null
+++ b/jena-db/dboe-base/src/main/java/org/seaborne/dboe/migrate/L.java
@@@ -1,0 -1,0 +1,206 @@@
++/*
++ *  Licensed under the Apache License, Version 2.0 (the "License");
++ *  you may not use this file except in compliance with the License.
++ *  You may obtain a copy of the License at
++ *
++ *      http://www.apache.org/licenses/LICENSE-2.0
++ *
++ *  Unless required by applicable law or agreed to in writing, software
++ *  distributed under the License is distributed on an "AS IS" BASIS,
++ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ *  See the License for the specific language governing permissions and
++ *  limitations under the License.
++ *
++ *  See the NOTICE file distributed with this work for additional
++ *  information regarding copyright ownership.
++ */
++
++package org.seaborne.dboe.migrate;
++
++import java.io.IOException ;
++import java.io.OutputStream ;
++import java.io.OutputStreamWriter ;
++import java.io.Writer ;
++import java.nio.ByteBuffer ;
++import java.nio.charset.StandardCharsets ;
++import java.util.UUID ;
++import java.util.concurrent.* ;
++import java.util.concurrent.locks.Lock ;
++import java.util.function.Supplier ;
++
++import org.apache.jena.atlas.io.IO ;
++import org.apache.jena.atlas.lib.Bytes ;
++import org.apache.jena.atlas.lib.StrUtils ;
++import org.apache.jena.shared.uuid.JenaUUID ;
++
++/** Misc class */
++public class L {
++    
++    // Not to be confused with UUID.nameUUIDFromBytes (a helper for version 3 UUIDs)
++    /**
++     * Java UUID to bytes (most significant first)
++     */
++    public static byte[] uuidAsBytes(UUID uuid) {
++        return uuidAsBytes(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()) ;
++    }
++    
++    /**
++     * Jena UUID to bytes (most significant first)
++     */
++    public static byte[] uuidAsBytes(JenaUUID uuid) {
++        return uuidAsBytes(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()) ;
++    }
++    
++    /** UUID, as two longs, as bytes */ 
++    public static byte[] uuidAsBytes(long mostSignificantBits, long leastSignificantBits) {
++        byte[] bytes = new byte[16] ;
++        Bytes.setLong(mostSignificantBits, bytes, 0); 
++        Bytes.setLong(leastSignificantBits, bytes, 8);
++        return bytes ;
++    }
++    
++    /** A UUID string to bytes */ 
++    public static byte[] uuidAsBytes(String str) {
++        return uuidAsBytes(UUID.fromString(str)) ;
++    }
++    
++    public static String uuidToString(long mostSignificantBits, long leastSignificantBits) {
++        return new UUID(mostSignificantBits, leastSignificantBits).toString() ;
++        //JenaUUID.toString(mostSignificantBits, leastSignificantBits)
++    }
++    
++    private static ExecutorService executor = Executors.newCachedThreadPool() ; 
++    
++    /**
++     * Run asynchronously on another thread ; the thread has started
++     * when this function returns.
++     */
++    public static void async(Runnable r) {
++        Semaphore semaStart = new Semaphore(0, true) ;
++        Runnable r2 = () -> {
++            semaStart.release(1) ;
++            r.run();
++        } ;
++        executor.execute(r2);
++        semaStart.acquireUninterruptibly();
++    }
++    
++    /** Run synchronously but on another thread. */
++    public static void syncOtherThread(Runnable r) {
++        runCallable(()->{
++            r.run();
++            return null ;
++        }) ;
++    }
++
++    /** Run synchronously but on another thread. */
++    public static <T> T syncCallThread(Supplier<T> r) {
++        return runCallable(() -> {
++            T t = r.get() ;
++            return t ;
++        }) ;
++    }
++    
++    private static <T> T runCallable(Callable<T> action) {
++        try { return executor.submit(action).get() ; }
++        catch (Exception e) {
++            e.printStackTrace();
++            return null ;
++        }
++    }
++    
++
++    /** Execute. Preform the "before" action, then main action.
++     *  Always call the "after" runnable if the "bafore" succeeded.
++     *  Becareful about argument order. 
++     * @param action
++     * @param before
++     * @param after
++     */
++    public static void withBeforeAfter(Runnable action, Runnable before, Runnable after) {
++        before.run(); 
++        try { action.run(); }
++        finally { after.run();  }
++    }
++
++    /** Execute. Preform the "before" action, then main action.
++     *  Always call the "after" runnable if the "bafore" succeeded.
++     *  Becareful about argument order. 
++     * @param action
++     * @param before
++     * @param after
++     */
++    public static <V> V callWithBeforeAfter(Supplier<V> action, Runnable before, Runnable after) {
++        before.run(); 
++        try { return action.get() ; } 
++        finally { after.run();  }
++    }
++
++    /** Execute; always call the "after" runnable */
++    public static void withAfter(Runnable action, Runnable after) {
++        try { action.run(); } 
++        finally { after.run();  }
++    }
++
++    /** Execute and return a value; always call the "after" runnable */
++    public static <V> V callWithAfter(Supplier<V> action, Runnable after) {
++        try { return action.get() ; }
++        finally { after.run();  }
++    }
++
++    /** Run inside a Lock */
++    public static  <V> V callWithLock(Lock lock, Supplier<V> r) {
++        return callWithBeforeAfter(r, ()->lock.lock(), ()->lock.unlock()) ;  
++    }
++    
++    /** Run inside a Lock */
++    public static void withLock(Lock lock, Runnable r) {
++        withBeforeAfter(r, ()->lock.lock(), ()->lock.unlock()) ;
++    }
++    
++    // ==> IO.writeWholeFileAsUTF8
++    
++    /** Write a string to a file as UTF-8. The file is closed after the operation.
++     * @param filename
++     * @param content String to be writtem
++     * @throws IOException
++     */
++    
++    public static void writeStringAsUTF8(String filename, String content) throws IOException {
++        try ( OutputStream out = IO.openOutputFileEx(filename) ) {
++            writeStringAsUTF8(out, content) ;
++            out.flush() ;
++        }
++    }
++
++    /** Read a whole stream as UTF-8
++     * 
++     * @param out       OutputStream to be read
++     * @param content   String to be written
++     * @throws  IOException
++     */
++    public static void writeStringAsUTF8(OutputStream out, String content) throws IOException {
++        Writer w = new OutputStreamWriter(out, StandardCharsets.UTF_8) ;
++        w.write(content);
++        w.flush();
++        // Not close.
++    }
++
++    // ==> IO.writeWholeFileAsUTF8
++    
++    /** String to ByteBuffer */
++    public static ByteBuffer stringToByteBuffer(String str) {
++        byte[] b = StrUtils.asUTF8bytes(str) ;
++        return ByteBuffer.wrap(b) ;
++    }
++    
++    /** ByteBuffer to String */
++    public static String byteBufferToString(ByteBuffer bb) {
++        byte[] b = new byte[bb.remaining()] ;
++        bb.get(b) ;
++        return StrUtils.fromUTF8bytes(b) ;
++    }
++
++
++}
++


Mime
View raw message