directory-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akaras...@apache.org
Subject svn commit: rev 6362 - in incubator/directory/snickers/trunk/ber: src/java/org/apache/snickers/asn src/java/org/apache/snickers/ber src/java/org/apache/snickers/test xdocs xdocs/empty
Date Fri, 30 Jan 2004 06:19:41 GMT
Author: akarasulu
Date: Thu Jan 29 22:19:41 2004
New Revision: 6362

Added:
   incubator/directory/snickers/trunk/ber/xdocs/
   incubator/directory/snickers/trunk/ber/xdocs/design.xml
   incubator/directory/snickers/trunk/ber/xdocs/empty/
   incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml
   incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml
   incubator/directory/snickers/trunk/ber/xdocs/eureka.xml
   incubator/directory/snickers/trunk/ber/xdocs/index.xml
   incubator/directory/snickers/trunk/ber/xdocs/navigation.xml
   incubator/directory/snickers/trunk/ber/xdocs/news.xml
   incubator/directory/snickers/trunk/ber/xdocs/rt.xml
Modified:
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/asn/BitString.java
  (props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/BerUtils.java 
 (props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java
  (contents, props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java   (contents,
props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java
  (contents, props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TypeClass.java
  (props changed)
   incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/test/TestBitString.java
  (props changed)
Log:
Added keyword expansion for author and revision and some baseline documentation
oh I also trimmed some lines down to 80 characters.


Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java
(original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/Primative.java
Thu Jan 29 22:19:41 2004
@@ -51,9 +51,11 @@
 package org.apache.snickers.ber;
 
 /**
- * @author Wes McKean
- *
  * Tag values for primitives
+ * 
+ * @author Wes McKean
+ * @author $Author$
+ * @version $Rev$
  */
 public interface Primative {
     public final int BOOLEAN = 1;

Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java	(original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLV.java	Thu Jan
29 22:19:41 2004
@@ -217,7 +217,8 @@
      * @return the id of this encoded value, or 0 if it doesn't have one
      */    
     public int getId() {
-        return ( tag & (Binary.BIT_4 | Binary.BIT_3 | Binary.BIT_2 | Binary.BIT_1 | Binary.BIT_0
) );
+        return ( tag & (Binary.BIT_4 | Binary.BIT_3 | Binary.BIT_2 | 
+                        Binary.BIT_1 | Binary.BIT_0 ) );
    }
    
    /**

Modified: incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java
==============================================================================
--- incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java
(original)
+++ incubator/directory/snickers/trunk/ber/src/java/org/apache/snickers/ber/TLVHandler.java
Thu Jan 29 22:19:41 2004
@@ -49,11 +49,13 @@
 */
 package org.apache.snickers.ber;
 
+
 import org.apache.commons.codec.DecoderException;
 
+
 /**
- * Basic interface for handling TLVs from the TLVParser.  The order of event firing
- * will normally (always) consist of:
+ * Basic interface for handling TLVs from the TLVParser.  The order of event 
+ * firing will normally (always) consist of:
  * 
  *      beginTag, length, data, received, endTag
  *
@@ -61,57 +63,76 @@
  * @author $Author$
  * @version $Rev$
  */
-public interface TLVHandler {
+public interface TLVHandler 
+{
     /**
      * Fired by the TLVParser to indicate that a new TLV is being decoded
      * 
-     * @param typeClass a type class object representing one of the four ASN.1 type classes
-     * @param id The encoded structure id.  Not all encoded values will have an associated
id
+     * @param typeClass a type class object representing one of the four ASN.1 
+     *      type classes
+     * @param id The encoded structure id.  Not all encoded values will have an 
+     *      associated id
      * @param isPrimative a boolean indicating that this value is a primitive
      * @throws DecoderException if an error occurs during handling
      */
-    public void beginTag( TypeClass typeClass, int id, boolean isPrimative ) throws DecoderException;
+    public void beginTag( TypeClass typeClass, int id, boolean isPrimative ) 
+        throws DecoderException;
     
     /**
-     * Fired by the TLVParser to indicate that the processing of the current TLV has been
completed
+     * Fired by the TLVParser to indicate that the processing of the current TLV
+     * has been completed
      * 
-     * @param typeClass a type class object representing one of the four ASN.1 type classes
-     * @param id The encoded structure id.  Not all encoded values will have an associated
id
+     * @param typeClass a type class object representing one of the four ASN.1 
+     *      type classes
+     * @param id The encoded structure id.  Not all encoded values will have 
+     *      an associated id
      * @throws DecoderException if an error occurs during handling
      */
     public void endTag( TypeClass typeClass, int id ) throws DecoderException;
 
     /**
-     * Fired by the TLVParser to indicate the length of the V portion of the TLV structure.
 Per the
-     * ASN.1 spec, there are three seperate encoding forms:  short, long, indefinite.  Please
note that
-     * the length will be zero if the form is indefinite.
+     * Fired by the TLVParser to indicate the length of the V portion of the 
+     * TLV structure.  Per the ASN.1 spec, there are three seperate encoding 
+     * forms:  short, long, indefinite.  Please note that the length will be 
+     * zero if the form is indefinite.
      * 
-     * @param typeClass a type class object representing one of the four ASN.1 type classes
-     * @param id The encoded structure id.  Not all encoded values will have an associated
id
+     * @param typeClass a type class object representing one of the four ASN.1 
+     *      type classes
+     * @param id The encoded structure id.  Not all encoded values will have 
+     *      an associated id
      * @param length the length of the V portion of the TLV structure
      * @throws DecoderException if an error occurs during handling
      */
-    public void length( TypeClass typeClass, int id, int length ) throws DecoderException;
+    public void length( TypeClass typeClass, int id, int length ) 
+        throws DecoderException;
     
     /**
-     * Fired by the TLVParser when data is received.  The data may be all of the required
length,
-     * or it may be coming over in chunks.  In other words, this event may be fired multiple
times.
+     * Fired by the TLVParser when data is received.  The data may be all of 
+     * the required length, or it may be coming over in chunks.  In other words,
+     * this event may be fired multiple times.
      * 
-     * @param typeClass a type class object representing one of the four ASN.1 type classes
-     * @param id The encoded structure id.  Not all encoded values will have an associated
id
+     * @param typeClass a type class object representing one of the four ASN.1 
+     *      type classes
+     * @param id The encoded structure id.  Not all encoded values will have an 
+     *      associated id
      * @param data the byte data associated with the value
      * @throws DecoderException if an error occurs during handling
      */
-    public void data( TypeClass typeClass, int id, byte[] data ) throws DecoderException;
+    public void data( TypeClass typeClass, int id, byte[] data ) 
+        throws DecoderException;
     
     /**
-     * Fired by the TLVParser once the entire TLV packet has been received.  This method
may
-     * need to be removed if we think it is going to be too much of a memory hog.
+     * Fired by the TLVParser once the entire TLV packet has been received.  
+     * This method may need to be removed if we think it is going to be too 
+     * much of a memory hog.
      * 
-     * @param typeClass a type class object representing one of the four ASN.1 type classes
-     * @param id The encoded structure id.  Not all encoded values will have an associated
id
+     * @param typeClass a type class object representing one of the four 
+     *      ASN.1 type classes
+     * @param id The encoded structure id.  Not all encoded values will have 
+     *      an associated id
      * @param tlv the TLV object which has just finished being parsed.
      * @throws DecoderException if an error occurs during handling
      */
-    public void received( TypeClass typeClass, int id, TLV tlv ) throws DecoderException;
+    public void received( TypeClass typeClass, int id, TLV tlv ) 
+        throws DecoderException;
 }

Added: incubator/directory/snickers/trunk/ber/xdocs/design.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/design.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Design Documentation</title>
+  </properties>
+  <body>
+    <section name="Design Documentation">
+      <p>Hang tight we'll be working this as we solidify our thoughts.</p>
+    </section>
+  </body>
+</document>
\ No newline at end of file

Added: incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/empty/index.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Empty!</title>
+  </properties>
+  <body>
+    <section name="Empty!">
+      <p>This is an empty place holder for pages one directory above.</p>
+    </section>
+  </body>
+</document>

Added: incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/empty/navigation.xml	Thu Jan 29 22:19:41
2004
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project>
+  <title>Apache Directory</title>
+  <body>
+    <links>
+      <item name="Apache" href="http://apache.org/"/>
+      <item name="Snickers" 
+        href="http://incubator.apache.org/directory/snickers"/>
+      <item name="Directory" href="http://incubator.apache.org/directory"/>
+    </links>
+
+    <menu name="About Snickers">
+      <item name="top item0" href="/empty/index.html"/>
+      <item name="top item1" href="/empty/index.html"/>
+      <item name="top item2" href="/empty/index.html"/>
+      <item name="BER Library" href="/index.html"/>
+      <item name="top item3" href="/empty/index.html"/>
+      <item name="top item4" href="/empty/index.html"/>
+    </menu>
+  </body>
+</project>

Added: incubator/directory/snickers/trunk/ber/xdocs/eureka.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/eureka.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,929 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Eureka!</title>
+  </properties> 
+  
+ <body>
+
+  <section name="And we saw the light ... ">
+     <p>
+       One day Wes and Alex started talking about going to town on a new 
+       ASN.1 BER Library and here's what happened ...
+     </p>    
+     <subsection name="The Conversation">
+      <source>
+[SNIP]
+
+Wes at Work says:
+I've been thinking about the decoding process a bit over the weekend.
+
+Alex Karasulu says:
+k I'm listening
+
+Wes at Work says:
+and encoding.
+
+Wes at Work says:
+I'm not sure at the initial stage there will be *one* decoder.
+
+Wes at Work says:
+We will need some place to hold our TLV tree.
+
+Wes at Work says:
+and also, I was thinking about really long messages.
+
+Alex Karasulu says:
+you need multiple codecs (coder decoders)
+
+Alex Karasulu says:
+right
+
+[SNIP]
+
+Wes at Work says:
+We got one part that builds the tree
+
+Wes at Work says:
+part two should be the translation.
+
+[SNIP]
+
+Wes at Work says:
+I think the only issue we have is how to handle chunking, and blocking versus 
+non-blocking code.
+
+Wes at Work says:
+And also, dealing with really huge messages.
+
+Wes at Work says:
+It obviously won't make sense to build a TLV tree in its entirety for a huge 
+search result.
+
+Alex Karasulu says:
+right I agree
+
+Alex Karasulu says:
+for encoding there is a mechanism for breaking down large TLVs of simple types 
+down 
+
+Wes at Work says:
+encoding is a non issue as far as chunking goes.
+
+Alex Karasulu says:
+basically in the book they talk about 3 ways of specifying length
+
+Alex Karasulu says:
+the L part
+
+Alex Karasulu says:
+right but it effects decoding
+
+Alex Karasulu says:
+but if another provider is doing encoding I see what you mean
+
+Alex Karasulu says:
+basically we can break stuff down by injecting the 3rd indeterminate length form
+
+Alex Karasulu says:
+follow me out
+
+Wes at Work says:
+You give the encoder an output interface, and every time it fills up the byte 
+bufffer, it spits it out.
+
+Alex Karasulu says:
+Strictly talking about decoding and chunk sizes for now.
+
+Wes at Work says:
+K.  decoding then.
+
+Alex Karasulu says:
+just for background - you read the section on the 3 different modes for 
+specifying length right: short, long and indeterminant?
+
+[SNIP]
+
+Alex Karasulu says:
+Your reading and encounter a really big simple type using the long encoding for 
+L.  So you know what you have to read is a hugh blob of data in one big hunk.  
+Basically there is some threashold u use to judge whether or not the blob is too
+big and needs to be chopped up.
+
+Wes at Work says:
+I did read the section the length.
+
+Alex Karasulu says:
+cool
+
+Wes at Work says:
+I actually printed the whole appendix out and read it.
+
+Wes at Work says:
+on BER.
+
+Alex Karasulu says:
+cool that's what I was reffering to
+
+Wes at Work says:
+An encoder can choose any one he wants.
+
+Alex Karasulu says:
+Now your decoder can break down the long format into the indeterminate format 
+nesting smaller TLVs inside the TLV.  Hence converting the simple TLV into a 
+constructed one.
+
+Alex Karasulu says:
+The key here is not to keep all the tlvs in memory or the entire encoded buffer 
+in memory
+
+Wes at Work says:
+For decoding, there are messages where keeping the intermediate form in memory 
+is not an issue, and with others, there are.
+
+Wes at Work says:
+issues.
+
+Alex Karasulu says:
+Right depends on the message size
+
+Wes at Work says:
+The client will want to process most of the messages as a complete object.
+
+Wes at Work says:
+By definition, it will be in memory.
+
+Alex Karasulu says:
+Yeah I know what you are saying.  We need to make the library not do this 
+though.  Then there would be more than one copy in memory.  Leave it upto the 
+user to determine how the data is dealt with.  Eventually we can take messures 
+to stream data if we want instead of having it all in memory.
+
+Wes at Work says:
+Back up just a second.
+
+Alex Karasulu says:
+There are funky tactics we can employ way down the road - but for the time 
+being lets make it so our codecs dont need massive footprints 
+
+Alex Karasulu says:
+sure talk to me
+
+Wes at Work says:
+I used this technique in a Btrieve interface I wrote for U. S. South...
+
+Wes at Work says:
+which I stole from OpenTDS.
+
+Alex Karasulu says:
+Btrieve?
+
+Wes at Work says:
+Yea, an ISAM database.
+
+Alex Karasulu says:
+oh ok
+
+Wes at Work says:
+It used byte buffers to send and retrieve records.
+
+Wes at Work says:
+I wrote a java class that basically treated the byte array as primitives.
+
+Alex Karasulu says:
+cool so you're already of the mindset to keeping the decoding and encoding 
+memory footprints small
+
+Wes at Work says:
+That might not work with us though.
+
+Wes at Work says:
+It might.
+
+Wes at Work says:
+All we need to know
+
+Wes at Work says:
+is that this field goes with this TLV.
+
+Wes at Work says:
+and convert it on the fly.
+
+Wes at Work says:
+Also, we an simply dump the TLVs when we are done.
+
+Alex Karasulu says:
+yeah that's part of some tables we may need to maintain with a mappiung
+
+Alex Karasulu says:
+right I think we're on the same page
+
+Alex Karasulu says:
+I have a small idea though
+
+Alex Karasulu says:
+Basically wrt the codec's interfaces
+
+Alex Karasulu says:
+To me you give an array of bytes in a byte[] or a ByteBuffer (this is the 
+delivered partial chunk) and you get back a set of TLVs for that chunk.
+
+Alex Karasulu says:
+or take it in the opposite direction for a encoder
+
+Alex Karasulu says:
+this is your stage 1 (BER bytes ->TLVs)
+
+Alex Karasulu says:
+now we need to find a way to represent TLVs in a linear fashion and still 
+maintain the tree structure.  However we don't want direct back references 
+to where the list of TLVs plug into the entire tree because this would mean 
+we have to have the whole tree in memory.
+
+Alex Karasulu says:
+does that make sense I know its a lil nebulous
+
+Wes at Work says:
+Keep it simple  
+
+Alex Karasulu says:
+ok in decoding bytes go in and TLVs come out
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+state is maintained between times u pump in bytes
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+Yup.
+
+Alex Karasulu says:
+now the TLVs comming out are a peice of the TLV tree
+
+Wes at Work says:
+You got to be able to handle partial Ts, Ls, and Vs.
+
+Alex Karasulu says:
+right that's part of the state stuff
+
+Alex Karasulu says:
+if you're stuck in the middle of a simple tlv then you don't pump it out until 
+the chunks to complete it have arrived
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+right.
+
+Alex Karasulu says:
+So the key here is to have the right TLV represntation or data structure.  We 
+have some requirements on this.
+
+Alex Karasulu says:
+the TLVs that come out of the decoder cannot directly, with java references, 
+refer to other TLVs  that came out before.  Because these references would 
+require the entire TLV tree in memory.
+
+Alex Karasulu says:
+This is one of those requirements you agree?
+
+Wes at Work says:
+I don't see that being an issue.
+
+Wes at Work says:
+The parent needs to know about the children, but not vis a versa.
+
+Alex Karasulu says:
+right
+
+Wes at Work says:
+and I don't see how you are going to be able to assemble an ASN.1 message in a 
+state driven fashion without making it very complicated.
+
+Alex Karasulu says:
+that's our primary issue here
+
+Wes at Work says:
+and have two decoders hooked together as well.
+
+Alex Karasulu says:
+its a big problem to overcome
+
+Alex Karasulu says:
+and do it elegantly
+
+Alex Karasulu says:
+If we do this then our BER ASN.1 codec will be hot working in a non-blocking 
+fashion and being very efficient.  It's like the way SAX is used for reading 
+XML for our ASN.1 messages instead of using DOM.
+
+Alex Karasulu says:
+the ideas are similar
+
+Alex Karasulu says:
+you didn't think this was gonna be a cake walk did ya  
+
+Wes at Work says:
+Hmmmm.
+
+Alex Karasulu says:
+you do understand where I was coming from wit the sax and dom stuff right?
+
+Wes at Work says:
+yea.
+
+Wes at Work says:
+That I understand.
+
+Alex Karasulu says:
+do you think its possible?
+
+Wes at Work says:
+So you have an event driven ASN.1 parser.
+
+Wes at Work says:
+I think that's still easy.
+
+Wes at Work says:
+However, assembling them into the messages is still complicated.
+
+Wes at Work says:
+every ASN.1 message type would have to be derived from our parser.
+
+Wes at Work says:
+Then a factory could create the message type based on the application type.
+
+Alex Karasulu says:
+hmmm
+
+Alex Karasulu says:
+what do you mean by: "every ASN.1 message type would have to be derived from 
+our parser.
+
+Wes at Work says:
+You want the ASN.1 messages to be able to assemble themselves? or no.
+
+Alex Karasulu says:
+Now you're talking about using the ASN.1 specification like a DTD to drive the 
+decoding
+
+Alex Karasulu says:
+?
+
+Alex Karasulu says:
+Yep I see yes
+
+Alex Karasulu says:
+u use the ASN.1 spec or a set of classes generated by an ASN.1 spec compiler
+
+Alex Karasulu says:
+question is do we need a compiler now?
+
+Wes at Work says:
+Right.
+
+Wes at Work says:
+Factory returns the ASN.1 message on the application tag.
+
+Alex Karasulu says:
+right I see where your going with the design
+
+Wes at Work says:
+the parser then passes everything to the ASN.reader interface,
+
+Wes at Work says:
+SAX like.
+
+Alex Karasulu says:
+Hmm sounds like it should be very possible
+
+Wes at Work says:
+of the application object.
+
+Wes at Work says:
+who knows how to assemble himself.
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+This is huge
+
+Alex Karasulu says:
+I wonder if other ASN.1 tools have this sax like mechanism already in place.
+
+Wes at Work says:
+But how do we handle ASN.1 messages which need to be streamed.
+
+Wes at Work says:
+like a huge search result.
+
+Alex Karasulu says:
+that's not so much the issue 
+
+Alex Karasulu says:
+a large result set takes n+2 messages
+
+Alex Karasulu says:
+sorry n+1
+
+Wes at Work says:
+You have a search result tight.
+
+Wes at Work says:
+Tag = Applicationz
+Length = 00
+Value = Search Results
+
+Wes at Work says:
+Now V is made up of thousands of result messages.
+
+Alex Karasulu says:
+In the LDAP protocol a search result is returned as n+1 messages.
+
+Alex Karasulu says:
+each result is an SearchEntryResponse for the 'n' and one SearchDoneResponse 
+PDU to end the resultset
+
+Alex Karasulu says:
+n+1 messages
+
+Wes at Work says:
+Ah.
+
+Wes at Work says:
+But are they wrapped in an application TLV?
+
+Alex Karasulu says:
+but think of a large blob of data
+
+Wes at Work says:
+or is it just one stream of TLVs.
+
+Alex Karasulu says:
+like say some binary chunk
+
+Alex Karasulu says:
+the application TLV for each response type is in the LDAP message envelope.  
+There is a top level LDAP message type which is a TLV then the different 
+response types have you know some enumeration values to determine which 
+response type the top level envelope or application TLV represents
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+but your question is valid for say a single SearchEntryResponse where one of 
+the attributes is a huge binary chunk
+
+Wes at Work says:
+So the event firing for the top level envelope will be different than the TLVs 
+which are part of the envelope.
+
+Alex Karasulu says:
+the top level LDAPMessage envelope defined for the LDAP asn.1 will be a 
+constructred TLV
+
+Alex Karasulu says:
+event might fire for it
+
+Alex Karasulu says:
+same one every time
+
+Wes at Work says:
+Right, but not after the entire TLV is read into memry.
+
+Wes at Work says:
+that would defeat our SAX based parser.
+
+Alex Karasulu says:
+but its constitution will change depending on the type of message it is
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+exactly
+
+Wes at Work says:
+I'm with you.
+
+Alex Karasulu says:
+you would get a start_ldap_message event
+
+Wes at Work says:
+Actually,
+
+Wes at Work says:
+for the envelope, you would need to hit the factory.
+
+Wes at Work says:
+to get the appropriate LDAP message.
+
+Alex Karasulu says:
+then perhaps the message_type_event will fire to note the contained TLV that 
+specifies the LDAP application's message type.
+
+Alex Karasulu says:
+et. cetera. see where i'm going with it - you don't need the entire message to 
+fire its arrival.  Like sax where you say start tag for this element then the 
+contained elemenets then close tags etc.
+
+Wes at Work says:
+Got ya.
+
+Wes at Work says:
+I think that's pretty cool.
+
+Alex Karasulu says:
+I think we're getting somewhere cool here I'm very excited.  I need to take 
+another look at a sax implementation again out there.  It will give me some 
+insight into some possible general architecture for us.
+
+Alex Karasulu says:
+Now going back to the massive chunk of binary.  So we have a 
+SearchEntryResponse with an entry of the result set containing an attribute 
+that is a huge binary chunk.  How do we stream it out right?  Then we can 
+talk about how we stream it in.
+
+Alex Karasulu says:
+Streaming it out is easy.  Let's for a moment presume that we can actually 
+stream out of the jdbm stuff.  You basically convert the long known length BER 
+encoding to the indeterminant encoding.  Then send out individual chunks of 
+this binary attribute in separate TLVs.  So you're turning big assed primitive 
+TLVs into constructed TLVs chunking out the content hence not needing the 
+entire V in memor
+
+Alex Karasulu says:
+y.
+
+Wes at Work says:
+That's fine for us.  We have control over the encoding.
+
+Wes at Work says:
+We won't be so lucky on the inbound side.
+
+Alex Karasulu says:
+Right
+
+Alex Karasulu says:
+Now let's think about that beast.
+
+Alex Karasulu says:
+We have a binary -> tlv encoder spitting out tlvs with each bit of input
+
+Alex Karasulu says:
+meant decoder above sorry
+
+Alex Karasulu says:
+now if the indeterminate length is used by the client when encoding and 
+sending to the server the server is ok the data is already chopped up and 
+its all good.  If not and the long length encoding is used then the data 
+comes into the server's decoder in chunks but the decoder sees a hugh 
+long length. 
+
+Alex Karasulu says:
+Based on some threshold the decoder translates the incoming long length and 
+values for the simple type (primitive TLV) into a constructed TLV breaking 
+up the large know length TLV into the indeterminant form which can be spit 
+out with a few nested TLVs at a time (with each input chunk going into the 
+decoder).
+
+Alex Karasulu says:
+You follow? Decoder automatically breaks up large primitive long length encoded 
+TLVs into the indeterminate form and spits those out in peices rather than the 
+one large primitive TLV.
+
+Wes at Work says:
+What does that buy us?
+
+Alex Karasulu says:
+streaming
+
+Wes at Work says:
+Is not the ASN message gonna re-assemble it anyways.
+
+Wes at Work says:
+Do you still end up with 200K picture in the ASN.1 message.
+
+Alex Karasulu says:
+yeah that's application specific - remember we're talking just the BER->TLV 
+codec
+
+Alex Karasulu says:
+the other codec is Type to TLV
+
+Wes at Work says:
+If we are using a SAX based parser, then the Type will be assembling itself as 
+the TLVs are decoded and fired.
+
+Alex Karasulu says:
+keeping it streaming means you don't have 2X the data or 400K in use just to 
+get the 200K picture
+
+Alex Karasulu says:
+right
+
+Wes at Work says:
+At some point, you are going to have to put your faith in the garbage collector.
+
+Alex Karasulu says:
+right but that's not in the codec BER to TLV code
+
+Alex Karasulu says:
+keep that lean and mean - why you ask
+
+Wes at Work says:
+Also, if you want a truly small memory footprint, then you could put stuff like 
+that in a small embedded database.
+
+Alex Karasulu says:
+well the TLV to Type code can be made lean and mean too
+
+Wes at Work says:
+I just don't think at this stage that we need to be all that worried about huge 
+blocks of binary data.
+
+Alex Karasulu says:
+right we use referrals to data on disk to manage large peices of data that 
+needsto be streamed but this we can do later.
+
+Wes at Work says:
+Exactly.
+
+Alex Karasulu says:
+yes but we want the options to be open - right now we can just design the 
+interfaces so all this can be added later.
+
+Alex Karasulu says:
+Interfaces and contracts should be designed to allow these very low memory 
+footprints.  Thinking through the process and what it takes to get there 
+makes us understand better what the design and interfaces should look like.
+
+Alex Karasulu says:
+I don't care if the first implementation is a hog
+
+Wes at Work says:
+The BER stuff today doesn't deal with this.
+
+Wes at Work says:
+It doesn't care.
+
+Alex Karasulu says:
+for large peices of data
+
+Wes at Work says:
+It's an application issue.
+
+Alex Karasulu says:
+right
+
+Alex Karasulu says:
+what the app does with it is upto the app but lets keep the ber codecs low in 
+memory image regardless of the fact that some app will be a pig and stream the 
+data into memory anyway.  This is all that I'm trying to say.
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+K.
+
+Alex Karasulu says:
+cool we're tight on this but I think it will take more research on both our 
+parts - anyway apache is back up again after a power failure.  Here's the new 
+stuff I created for ya:   
+http://cvs.apache.org/viewcvs.cgi/incubator/directory/snickers/?root=Apache-SVN
+
+Alex Karasulu says:
+that's the top level of the snickers (snacc replacement) subproject
+
+Alex Karasulu says:
+that's all you and Jeff with the C based version of this thang
+
+Wes at Work says:
+Right.
+
+Wes at Work says:
+You won't find much other ASN.1 stuff out there.
+
+Wes at Work says:
+I'm comfortable that no one is doing it this way, either.
+
+Wes at Work says:
+It will make it unqiuely, Apache.
+
+Alex Karasulu says:
+Ok. Let's touch base in a day or two to regroup
+
+Wes at Work says:
+Do you think ASN.1 is going to die?
+
+Alex Karasulu says:
+this is all good stuff and I'll try to get it out there.
+
+Alex Karasulu says:
+no way
+
+Alex Karasulu says:
+ASN.1 is awesome stuff
+
+Wes at Work says:
+We'll see.
+
+Alex Karasulu says:
+SNMP is based on it and so is Kerberose
+
+Alex Karasulu says:
+what's the alternative?
+
+Wes at Work says:
+XML is what everyone is using now.
+
+Alex Karasulu says:
+well there is XER for ASN.1 
+
+Alex Karasulu says:
+XML Encoding Rules
+
+Alex Karasulu says:
+ASN.1 can go to BER, PER, XER, and DER
+
+Wes at Work says:
+Yes.
+
+Alex Karasulu says:
+the encoding does not effect the ASN.1 specification and that is what makes 
+ASN.1 a winner always.
+
+Wes at Work says:
+Slapping XML on ASN.1 ain't the same.
+
+Alex Karasulu says:
+the XML format is just for the encoding of the data types 
+
+Wes at Work says:
+I agree that ASN.1 is a good protocol.
+
+Alex Karasulu says:
+protocol specification syntax
+
+Alex Karasulu says:
+it kicks ass I think and is here to stay.
+
+Wes at Work says:
+If we do this, we are going to go backwards right?
+
+Wes at Work says:
+Do the compiler last.
+
+Alex Karasulu says:
+go backwards?
+
+Alex Karasulu says:
+yeah that might be the case or we can work it together.
+
+Wes at Work says:
+You need to let me work this.
+
+Alex Karasulu says:
+I can do the compiler with you and you can handle the runtime
+
+Wes at Work says:
+You got other things to do.
+
+Alex Karasulu says:
+ok its all you then 
+
+Alex Karasulu says:
+I'm just a follower
+
+Wes at Work says:
+I won't mind help with the compiler.
+
+Wes at Work says:
+Just don't get going on it any time soon  
+
+Alex Karasulu says:
+sure I have extensive javacc and antlr experience
+
+Wes at Work says:
+Deal.
+
+Alex Karasulu says:
+hehe no worries with that my plate as you know is overflowing.
+
+Alex Karasulu says:
+my bladder too
+
+Alex Karasulu says:
+I'll catch ya later I need to hit the head
+
+Wes at Work says:
+Talk about the decoder's stream.
+
+Wes at Work says:
+K
+
+Alex Karasulu says:
+ttyl
+
+Wes at Work says:
+Talk later then.
+
+Alex Karasulu says:
+ok gimme 45 seconds
+
+Alex Karasulu says:
+I'm back
+
+Alex Karasulu says:
+what about the decoder's stream.
+
+Wes at Work says:
+So, how do we feed the decoder then.
+
+Alex Karasulu says:
+Its all about how we design our interfaces.  You know I've been looking at 
+commons-codec and see some potential but changes will be needed.
+
+Alex Karasulu says:
+Follow me for a sec.
+
+Alex Karasulu says:
+Now the codec interfaces are designed to convert stuff in one shot.  
+bytes in bytes out sort of thang.  Very blocking dependent stuff and not very 
+cool for us with a SEDA and NIO based server.  
+
+Alex Karasulu says:
+wit me?
+
+Wes at Work says:
+right.
+
+Alex Karasulu says:
+As you might have guessed this is not good for servers that need to keep 
+memory footprints low while servicing possible serveral hundred requests 
+per second.
+
+Alex Karasulu says:
+So what do we do? We design new non-blocking and NIO based interfaces for the 
+codec API and submit them.  
+
+Alex Karasulu says:
+its down again damn
+
+Wes at Work says:
+I got my update  
+
+Alex Karasulu says:
+cool
+
+Wes at Work says:
+Must of brought it down.
+
+Alex Karasulu says:
+yeah maybe it will be up soon
+
+Alex Karasulu says:
+anyway
+
+Alex Karasulu says:
+We redesign these codec interfaces to manage an encoding session and a decoding
+session so chunks can be process in a stateful manner to be conducive to 
+non-blocking use.
+
+Alex Karasulu says:
+Or we use events like you said
+
+Alex Karasulu says:
+Basically we contribute this to the commons stuff and make sure the community 
+understands why and what we're doing.  That way they can double check us.
+
+Alex Karasulu says:
+Then we use those interfaces to implement the ASN.1 stuff.
+
+Wes at Work says:
+Right.
+
+Alex Karasulu says:
+We do this in the snickers area but put back as much into the commons codec as 
+we can.  You game with this strategy?
+
+Wes at Work says:
+Yea, that's fine.
+
+Wes at Work says:
+I'll check out commons code as soon as it comes up.      
+
+</source>
+     </subsection>
+   </section>
+  </body>
+</document>

Added: incubator/directory/snickers/trunk/ber/xdocs/index.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/index.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Snickers ASN.1 BER Library</title>
+  </properties>
+  <body>
+    <section name="News and Information">
+      <ul>
+        <li>
+	        26-JAN-2004: <a href="news.html">Created Snickers Sub-Project</a>
+        </li>
+      </ul>
+    </section>
+    <section name="What is it and why is it called Snickers?">
+      <p>
+        The Basic Encoding Rules (BER) Library is a simple encoding for Abstract
+        Syntax Notation One (ASN.1) defined data structures.  It replaces the
+        SNACC4J runtime libraries and adds some enhancements.  It's designed
+        for efficiency and speed specifically for use in SEDA based protocol 
+        servers using a BER codec for their Protocol Data Units (PDU).
+      </p>
+      <p>
+        Originally we used the SNACC4J compiler to generate stubs that knew how
+        to encode and decode themselves.  Due to licensing issues we had to 
+        abandon the use of SNACC4J.  Rather than reimplement it we decided to 
+        innovate on top of the algorithms used by Sample and Neufeld (the 
+        original creators of the first free SNAC compiler).  
+      </p>
+      <p>
+        A member of the directory team, Robb Penoyer, thought it was funny to
+        call whatever replaced the SNACC4J runtime, Snickers.  We think he was
+        hungry when he came up with this one.  Since then the name Snickers has 
+        stuck to the belly of the directory project and refers to everything 
+        dealing with ASN.1.
+      </p>
+    </section>
+    <section name="How will it work?">
+      <p>
+        The BER encoder and decoder (codec) will be designed to operate very
+        much like the Simple API for XML (SAX).  It will generate events which
+        are calls on a handler as it encounters low level encoded structures
+        called Tag Length Value (TLV) triplets.  
+      </p>
+      <p>
+        Rather than return values, which could be extremely large, in one peice
+        the decoder for example returns peices of a value until it completes 
+        processing the entire V of the TLV.  This makes the decoder highly 
+        attractive for servers using non-blocking IO and SocketChannels.  This
+        design gives it a small decoding footprint regardless of the size of 
+        the Protocol Data Unit (PDU) being processed.  It also makes it much
+        faster since the decoder deals with a small simple task without much
+        conditional logic for processing a PDU.  We hope the combined benefits
+        of non-blocking IO and this sleek codec, make any BER based protocol
+        server extremely responsive under heavy loads with massive concurrency.
+      </p>
+    </section>
+    <section name="How to use the API">
+      <p>
+        Fill this in once we have an API.
+      </p>
+      <subsection name="Some Sub-Topic">
+        <p>
+          Fill this in.
+        </p>
+      </subsection>
+      <subsection name="General Resources">
+        <ul>
+          <li><a href="rt.html">Random Thoughts</a></li>
+          <li><a href="eureka.html">Eureka!</a></li>
+        </ul>
+      </subsection>
+    </section>
+  </body>
+</document>

Added: incubator/directory/snickers/trunk/ber/xdocs/navigation.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/navigation.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project>
+  <title>Apache Directory</title>
+  <body>
+    <links>
+      <item name="Apache" href="http://apache.org/"/>
+      <item name="Snickers" 
+        href="http://incubator.apache.org/directory/snickers"/>
+      <item name="Directory" href="http://incubator.apache.org/directory"/>
+    </links>
+    <menu name="About Snickers">
+      <item name="top item0" href="/empty/index.html"/>
+      <item name="top item1" href="/empty/index.html"/>
+      <item name="top item2" href="/empty/index.html"/>
+      <item name="BER Library" href="/index.html">
+        <item name="Overview" href="/index.html"/>
+        <item name="Design" href="/design.html"/>
+        <item name="Latest News" href="/news.html"/>
+        <item name="Random Thoughts" href="/rt.html"/>
+        <item name="Eureka" href="/eureka.html"/>
+        <item name="Sample's Original SNACC" 
+          href="http://www.apache.org/~akarasulu/resources/ASN.1/snacc.pdf"/>
+      </item>
+      <item name="top item3" href="/empty/index.html"/>
+      <item name="top item4" href="/empty/index.html"/>
+    </menu>
+  </body>
+</project>

Added: incubator/directory/snickers/trunk/ber/xdocs/news.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/news.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Snickers BER Library News</title>
+  </properties> 
+  
+ <body>
+
+  <section name="News and Information">
+
+     <!-- insert new news items as a subsection here -->
+
+     <subsection name="26-JAN-2004: Created Snickers Sub-Project">
+      <p> We realized that IBM's SNACC just wont do a long time ago due to 
+        licensing restrictions. Anyway IBM had taken the original SNACC 
+        algorithm which was open and closed the source so it grew cobwebs. It 
+        was time for something better. We soon came to the realization that we 
+        would have to due better than just a reimplementation of the original 
+        SNACC generated codecs. </p>
+      <p>
+        BTW SNACC is an acronym for the Sample, Neufeld ASN.1 to C/C++ compiler.
+        Although this part of snickers is not an ASN.1 to C/C++ compiler it does
+        deal with the part of the generated code that decodes and encodes ASN.1
+        based data structures to and from the basic encoding.
+      </p>
+      <p>
+        The problem with the snacc approach to encoding and decoding ASN.1 
+        structures rests in the fact that an entire copy of the structure must 
+        be kept in memory while it is encoded and finishes decoding.  Still the
+        processing may require even more memory while handling this process 
+        depending on the TLV count to average value size ratio of the PDU.
+      </p>
+      <p>
+        We decided to devise our own mechanism for encoding and decoding BER 
+        based PDUs.  Our approach minimizes the memory footprints while decoding
+        and encoding messages without requiring the entire memory footprint of
+        the PDU in memory.  This is what gave birth to the snickers project.
+      </p>
+     </subsection>
+
+   </section>
+  </body>
+</document>

Added: incubator/directory/snickers/trunk/ber/xdocs/rt.xml
==============================================================================
--- (empty file)
+++ incubator/directory/snickers/trunk/ber/xdocs/rt.xml	Thu Jan 29 22:19:41 2004
@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <properties>
+    <author email="akarasulu@apache.org">Alex Karasulu</author>
+    <title>Random Thoughts</title>
+  </properties> 
+  
+ <body>
+  <section name="Wes' Random Thoughts">
+     <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+      <p>
+        Put em here!
+      </p>
+     </subsection>
+   </section>
+  <section name="Alex's Random Thoughts">
+     <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+      <p>
+        Put em here!
+      </p>
+     </subsection>
+   </section>
+  <section name="Jeff's Random Thoughts">
+     <subsection name="26-JAN-2004: Some Thoughts Here :-)">
+      <p>
+        Put em here!
+      </p>
+     </subsection>
+   </section>
+  </body>
+</document>

Mime
View raw message