camel-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Claus Ibsen (Confluence)" <conflue...@apache.org>
Subject [CONF] Apache Camel > HDFS2
Date Thu, 13 Mar 2014 09:24:00 GMT
<html>
    <head>
        <meta name="viewport" content="width=device-width" />
        <base href="https://cwiki.apache.org/confluence" />
        <style type="text/css">
    body, #email-content, #email-content-inner { font-family: Arial,FreeSans,Helvetica,sans-serif;
}
    body, p, blockquote, pre, code, td, th, li, dt, dd { font-size: 13px; }
    small { font-size: 11px; }

    body { width:100% !important; -webkit-font-smoothing: antialiased; }

    body,
    #email-wrapper { background-color: #f0f0f0; }
    #email-wrapper-inner { padding: 20px; text-align: center; }
    #email-content-inner { background-color: #fff; border: 1px solid #bbb; color: $menuTxtColour;
padding:20px; text-align:left; }
    #email-wrapper-inner > table { width: 100%; }
    #email-wrapper-inner.thin > table { margin: 0 auto; width: 50%; }
    #email-footer { padding: 0 16px 32px 16px; margin: 0; }

    .email-indent { margin: 8px 0 16px 0; }
    .email-comment { margin: 0 0 0 56px; }
    .email-comment.removed { background-color: #ffe7e7; border: 1px solid #df9898; padding:
0 8px;}

    #email-title-avatar { text-align: left; vertical-align: top; width: 48px; padding-right:
8px; }
    #email-title-flavor { margin: 0; padding: 0 0 4px 0; }
    #email-title-heading { font-size: 16px; line-height: 20px; min-height: 20px; margin: 0;
padding: 0; }
    #email-title .icon { border: 0; padding: 0 5px 0 0; text-align: left; vertical-align:
middle; }

    #email-actions { border-top: 1px solid #bbb; color: #505050; margin: 8px 0 0 0; padding:
0; }
    #email-actions td { padding-top: 8px; }
    #email-actions .left { max-width: 45%; text-align: left; }
    #email-actions .right { text-align: right; }
    .email-reply-divider { border-top: 1px solid #bbb; color: #505050; margin: 32px 0 8px
0; padding: 8px 0; }
    .email-section-title { border-bottom: 1px solid #bbb; margin: 8px 0; padding: 8px 0 0
0; }

    .email-metadata { color: #505050; }

    a { color: #326ca6; text-decoration: none; }
    a:hover { color: #336ca6; text-decoration: underline; }
    a:active {color: #326ca6; }

    a.email-footer-link { color: #505050; font-size: 11px; }

    .email-item-list { list-style: none; margin: 4px 0; padding-left: 0; }
    .email-item-list li { list-style: none; margin: 0; padding: 4px 0; }
    .email-list-divider { color: #505050; padding: 0 0.35em; }
    .email-operation-icon { padding-right: 5px; }

    .avatar { -ms-interpolation-mode: bicubic; border-radius: 3px;}
    .avatar-link { margin: 2px; }

    .tableview th { border-bottom: 1px solid #69C; font-weight: bold; text-align: left; }
    .tableview td { border-bottom: 1px solid #bbbbbb; text-align: left; padding: 4px 16px
4px 0; }

    .aui-message {  margin: 1em 0; padding: 8px; }
    .aui-message.info { background-color: #e0f0ff; border: 1px solid #9eb6d4; }
    .aui-message.success { background-color: #ddfade; border: 1px solid #93c49f; }
    .aui-message.error,
    .aui-message.removed { background-color: #ffe7e7; border: 1px solid #df9898; color: #000;
}

    .call-to-action-table { margin: 10px 1px 1px 1px;}
    .call-to-cancel-container, .call-to-action-container { padding: 5px 20px; }
    .call-to-cancel-container { border: 1px solid #aaa; background-color: #eee; border-radius:
3px; }
    .call-to-cancel-container a.call-to-cancel-button { background-color: #eee; font-size:
14px; line-height: 1; padding: 0; margin: 0; color: #666; font-family: sans-serif;}
    .call-to-action-container { border: 1px solid #486582;  background-color: #3068A2; border-radius:
3px; padding: 4px 10px; }
    .call-to-action-container a.call-to-action-button { background-color: #3068A2; font-size:
14px; line-height: 1; padding: 0; margin: 0; color: #fff; font-weight: bold; font-family:
sans-serif; }

    /** The span around the inline task checkbox image */
    .diff-inline-task-overlay {
        display: inline-block;
        text-align: center;
        height: 1.5em;
        padding: 5px 0px 1px 5px;
        margin-right: 5px;
        /** Unfortunately, the negative margin-left is stripped out in gmail */
        margin-left: -5px;
    }

            @media handheld, only screen and (max-device-width: 480px) {
        div, a, p, td, th, li, dt, dd { -webkit-text-size-adjust: auto; }
        small, small a { -webkit-text-size-adjust: 90%; }

        td[id=email-wrapper-inner] { padding: 2px !important; }
        td[id=email-content-inner] { padding: 8px !important; }
        td[id="email-wrapper-inner"][class="thin"] > table { text-align: left !important;
width: 100% !important; }
        td[id=email-footer] { padding: 8px 12px !important; }
        div[class=email-indent] { margin: 8px 0px !important; }
        div[class=email-comment] { margin: 0 !important; }

        p[id=email-title-flavor] a { display: block; } /* puts the username and the action
on separate lines */
        p[id=email-permalink] { padding: 4px 0 0 0 !important; }

        table[id=email-actions] td { padding-top: 0 !important; }
        table[id=email-actions] td.right { text-align: right !important; }
        table[id=email-actions] .email-list-item { display: block; margin: 1em 0 !important;
word-wrap: normal !important; }
        span[class=email-list-divider] { display: none; }
    }



        </style>
    </head>
    <body style="font-family: Arial, FreeSans, Helvetica, sans-serif; font-size: 13px;
width: 100%; -webkit-font-smoothing: antialiased; background-color: #f0f0f0">
        <table id="email-wrapper" width="100%" cellspacing="0" cellpadding="0" border="0"
style="background-color: #f0f0f0">
            <tbody>
                <tr valign="middle">
                    <td id="email-wrapper-inner" style="font-size: 13px; padding: 20px;
text-align: center">
                        <table id="email-content" cellspacing="0" cellpadding="0" border="0"
style="font-family: Arial, FreeSans, Helvetica, sans-serif; width: 100%">
                            <tbody>
                                <tr valign="top">
                                    <td id="email-content-inner" align="left" style="font-family:
Arial, FreeSans, Helvetica, sans-serif; font-size: 13px; background-color: #fff; border: 1px
solid #bbb; padding: 20px; text-align: left">
                                        <table id="email-title" cellpadding="0" cellspacing="0"
border="0" width="100%">
                                            <tbody>
                                                <tr>
                                                    <td id="email-title-avatar" rowspan="2"
style="font-size: 13px; text-align: left; vertical-align: top; width: 48px; padding-right:
8px"> <img class="avatar" src="cid:avatar_0bcbc540a20cfb82d3bb3f7e27fda8e6" border="0"
height="48" width="48" style="-ms-interpolation-mode: bicubic; border-radius: 3px" /> </td>
                                                    <td valign="top" style="font-size:
13px">
                                                        <div id="email-title-flavor" class="email-metadata"
style="margin: 0; padding: 0 0 4px 0; color: #505050">
                                                            <a href="    https://cwiki.apache.org/confluence/display/~davsclaus
" style="color:#326ca6;text-decoration:none;; color: #326ca6; text-decoration: none">Claus
Ibsen</a> created a page:
                                                        </div> </td>
                                                </tr>
                                                <tr>
                                                    <td valign="top" style="font-size:
13px"> <h2 id="email-title-heading" style="font-size: 16px; line-height: 20px; min-height:
20px; margin: 0; padding: 0"> <a href="https://cwiki.apache.org/confluence/display/CAMEL/HDFS2"
style="color: #326ca6; text-decoration: none"> <img class="icon" src="cid:page-icon"
alt="" style="border: 0; padding: 0 5px 0 0; text-align: left; vertical-align: middle" />
<strong style="font-size:16px;line-height:20px;vertical-align:top;">HDFS2</strong>
</a> </h2> </td>
                                                </tr>
                                            </tbody>
                                        </table>
                                        <div class="email-indent" style="margin: 8px 0
16px 0">
                                            <div class="email-page">
                                                <h2 id="HDFS2-HDFS2Component">HDFS2
Component</h2>
                                                <p style="font-size: 13px"> <strong>Available
as of Camel 2.13</strong> </p>
                                                <p style="font-size: 13px">The <strong>hdfs2</strong>
component enables you to read and write messages from/to an HDFS file system using Hadoop
2.x. HDFS is the distributed file system at the heart of <a href="http://hadoop.apache.org"
class="external-link" rel="nofollow" style="color: #326ca6; text-decoration: none">Hadoop</a>.</p>
                                                <p style="font-size: 13px">Maven users
will need to add the following dependency to their <code style="font-size: 13px">pom.xml</code>
for this component:</p>
                                                <div class="code panel pdl" style="border-width:
1px;">
                                                    <div class="codeContent panelContent
pdl">
                                                        <pre class="theme: Default; brush:
xml; gutter: false" style="font-size:12px;; font-size: 13px">&lt;dependency&gt;
    &lt;groupId&gt;org.apache.camel&lt;/groupId&gt;
    &lt;artifactId&gt;camel-hdfs2&lt;/artifactId&gt;
    &lt;version&gt;x.x.x&lt;/version&gt;
    &lt;!-- use the same version as your Camel core version --&gt;
&lt;/dependency&gt;
</pre>
                                                    </div>
                                                </div>
                                                <h3 id="HDFS2-URIformat">URI format</h3>
                                                <div class="code panel pdl" style="border-width:
1px;">
                                                    <div class="codeContent panelContent
pdl">
                                                        <pre class="theme: Default; brush:
java; gutter: false" style="font-size:12px;; font-size: 13px">hdfs2://hostname[:port][/path][?options]
</pre>
                                                    </div>
                                                </div>
                                                <p style="font-size: 13px">You can append
query options to the URI in the following format, <code style="font-size: 13px">?option=value&amp;option=value&amp;...</code>
<br /> The path is treated in the following way:</p>
                                                <ol>
                                                    <li style="font-size: 13px">as a
consumer, if it's a file, it just reads the file, otherwise if it represents a directory it
scans all the file under the path satisfying the configured pattern. All the files under that
directory must be of the same type.</li>
                                                    <li style="font-size: 13px">as a
producer, if at least one split strategy is defined, the path is considered a directory and
under that directory the producer creates a different file per split named using the configured
<a href="https://cwiki.apache.org/confluence/display/CAMEL/UuidGenerator" style="color:
#326ca6; text-decoration: none">UuidGenerator</a>.</li>
                                                </ol>
                                                <h3 id="HDFS2-Options">Options</h3>
                                                <div class="confluenceTableSmall">
                                                    <table class="confluenceTable" style="border-collapse:
collapse; border: 1px solid #ddd;">
                                                        <tbody>
                                                            <tr>
                                                                <th class="confluenceTh"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;background-color: #F0F0F0; font-weight: bold;; font-size: 13px"> <p style="font-size:
13px"> Name </p> </th>
                                                                <th class="confluenceTh"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;background-color: #F0F0F0; font-weight: bold;; font-size: 13px"> <p style="font-size:
13px"> Default Value </p> </th>
                                                                <th class="confluenceTh"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;background-color: #F0F0F0; font-weight: bold;; font-size: 13px"> <p style="font-size:
13px"> Description </p> </th>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">overwrite</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">true</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The file can be overwritten </p>
</td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">append</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">false</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> Append to existing file. Notice
that not all HDFS file systems support the append option. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">bufferSize</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">4096</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The buffer size used by HDFS
</p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">replication</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">3</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The HDFS replication factor </p>
</td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">blockSize</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">67108864</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The size of the HDFS blocks </p>
</td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">fileType</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">NORMAL_FILE</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> It can be SEQUENCE_FILE, MAP_FILE,
ARRAY_FILE, or BLOOMMAP_FILE, see Hadoop </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">fileSystemType</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">HDFS</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> It can be LOCAL for local filesystem
</p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">keyType</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">NULL</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The type for the key in case
of sequence or map files. See below. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">valueType</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">TEXT</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The type for the key in case
of sequence or map files. See below. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">splitStrategy</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px">&nbsp;</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> A string describing the strategy
on how to split the file based on different criteria. See below. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">openedSuffix</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">opened</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> When a file is opened for reading/writing
the file is renamed with this suffix to avoid to read it during the writing phase. </p>
</td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">readSuffix</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">read</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> Once the file has been read is
renamed with this suffix to avoid to read it again. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">initialDelay</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">0</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> For the consumer, how much to
wait (milliseconds) before to start scanning the directory. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">delay</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">0</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The interval (milliseconds) between
the directory scans. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">pattern</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">*</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> The pattern used for scanning
the directory </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">chunkSize</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">4096</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> When reading a normal file, this
is split into chunks producing a message per chunk. </p> </td>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">connectOnStartup</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">true</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <strong>Camel 2.9.3/2.10.1:</strong>
Whether to connect to the HDFS file system on starting the producer/consumer. If <code
style="font-size: 13px">false</code> then the connection is created on-demand. Notice
that HDFS may take up till 15 minutes to establish a connection, as it has hardcoded 45 x
20 sec redelivery. By setting this option to <code style="font-size: 13px">false</code>
allows your application to startup, and not block for up till 15 minutes. </p> </td>
                                                            </tr>
                                                        </tbody>
                                                    </table>
                                                </div>
                                                <h4 id="HDFS2-KeyTypeandValueType">KeyType
and ValueType</h4>
                                                <ul>
                                                    <li style="font-size: 13px">NULL
it means that the key or the value is absent</li>
                                                    <li style="font-size: 13px">BYTE
for writing a byte, the java Byte class is mapped into a BYTE</li>
                                                    <li style="font-size: 13px">BYTES
for writing a sequence of bytes. It maps the java ByteBuffer class</li>
                                                    <li style="font-size: 13px">INT
for writing java integer</li>
                                                    <li style="font-size: 13px">FLOAT
for writing java float</li>
                                                    <li style="font-size: 13px">LONG
for writing java long</li>
                                                    <li style="font-size: 13px">DOUBLE
for writing java double</li>
                                                    <li style="font-size: 13px">TEXT
for writing java strings</li>
                                                </ul>
                                                <p style="font-size: 13px">BYTES is
also used with everything else, for example, in Camel a file is sent around as an InputStream,
int this case is written in a sequence file or a map file as a sequence of bytes.</p>
                                                <h3 id="HDFS2-SplittingStrategy">Splitting
Strategy</h3>
                                                <p style="font-size: 13px">In the current
version of Hadoop opening a file in append mode is disabled since it's not very reliable.
So, for the moment, it's only possible to create new files. The Camel HDFS endpoint tries
to solve this problem in this way:</p>
                                                <ul>
                                                    <li style="font-size: 13px">If the
split strategy option has been defined, the hdfs path will be used as a directory and files
will be created using the configured <a href="https://cwiki.apache.org/confluence/display/CAMEL/UuidGenerator"
style="color: #326ca6; text-decoration: none">UuidGenerator</a> </li>
                                                    <li style="font-size: 13px">Every
time a splitting condition is met, a new file is created.<br /> The splitStrategy option
is defined as a string with the following syntax:<br /> splitStrategy=&lt;ST&gt;:&lt;value&gt;,&lt;ST&gt;:&lt;value&gt;,*</li>
                                                </ul>
                                                <p style="font-size: 13px">where &lt;ST&gt;
can be:</p>
                                                <ul>
                                                    <li style="font-size: 13px">BYTES
a new file is created, and the old is closed when the number of written bytes is more than
&lt;value&gt;</li>
                                                    <li style="font-size: 13px">MESSAGES
a new file is created, and the old is closed when the number of written messages is more than
&lt;value&gt;</li>
                                                    <li style="font-size: 13px">IDLE
a new file is created, and the old is closed when no writing happened in the last &lt;value&gt;
milliseconds</li>
                                                </ul>
                                                <div class="aui-message warning shadowed
information-macro" style="margin: 1em 0; padding: 8px">
                                                    <span class="aui-icon icon-warning">Icon</span>
                                                    <div class="message-content">
                                                        <p style="font-size: 13px">note
that this strategy currently requires either setting an IDLE value or setting the HdfsConstants.HDFS_CLOSE
header to false to use the BYTES/MESSAGES configuration...otherwise, the file will be closed
with each message</p>
                                                    </div>
                                                </div>
                                                <p style="font-size: 13px">for example:</p>
                                                <div class="code panel pdl" style="border-width:
1px;">
                                                    <div class="codeContent panelContent
pdl">
                                                        <pre class="theme: Default; brush:
java; gutter: false" style="font-size:12px;; font-size: 13px">hdfs://localhost/tmp/simple-file?splitStrategy=IDLE:1000,BYTES:5
</pre>
                                                    </div>
                                                </div>
                                                <p style="font-size: 13px">it means:
a new file is created either when it has been idle for more than 1 second or if more than
5 bytes have been written. So, running <code style="font-size: 13px">hadoop fs -ls /tmp/simple-file</code>
you'll see that multiple files have been created.</p>
                                                <h3 id="HDFS2-MessageHeaders">Message
Headers</h3>
                                                <p style="font-size: 13px">The following
headers are supported by this component:</p>
                                                <h4 id="HDFS2-Produceronly">Producer
only</h4>
                                                <div class="confluenceTableSmall">
                                                    <table class="confluenceTable" style="border-collapse:
collapse; border: 1px solid #ddd;">
                                                        <tbody>
                                                            <tr>
                                                                <th class="confluenceTh"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;background-color: #F0F0F0; font-weight: bold;; font-size: 13px"> <p style="font-size:
13px"> Header </p> </th>
                                                                <th class="confluenceTh"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;background-color: #F0F0F0; font-weight: bold;; font-size: 13px"> <p style="font-size:
13px"> Description </p> </th>
                                                            </tr>
                                                            <tr>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <code style="font-size: 13px">CamelFileName</code>
</p> </td>
                                                                <td class="confluenceTd"
style="border: 1px solid #DDD; padding: 5px 7px; min-width: 0.6em; text-align: left; vertical-align:
top;; font-size: 13px"> <p style="font-size: 13px"> <strong>Camel 2.13:</strong>
Specifies the name of the file to write (relative to the endpoint path). The name can be a
<code style="font-size: 13px">String</code> or an <a href="/confluence/display/CAMEL/Expression"
title="Expression" style="color: #326ca6; text-decoration: none">Expression</a> object.
Only relevant when not using a split strategy. </p> </td>
                                                            </tr>
                                                        </tbody>
                                                    </table>
                                                </div>
                                                <h3 id="HDFS2-Controllingtoclosefilestream">Controlling
to close file stream</h3>
                                                <p style="font-size: 13px">When using
the <a href="https://cwiki.apache.org/confluence/display/CAMEL/HDFS2" style="color: #326ca6;
text-decoration: none">HDFS2</a> producer <strong>without</strong> a
split strategy, then the file output stream is by default closed after the write. However
you may want to keep the stream open, and only explicitly close the stream later. For that
you can use the header <code style="font-size: 13px">HdfsConstants.HDFS_CLOSE</code>
(value = <code style="font-size: 13px">&quot;CamelHdfsClose&quot;</code>)
to control this. Setting this value to a boolean allows you to explicit control whether the
stream should be closed or not.</p>
                                                <p style="font-size: 13px">Notice this
does not apply if you use a split strategy, as there are various strategies that can control
when the stream is closed.</p>
                                                <h3 id="HDFS2-UsingthiscomponentinOSGi">Using
this component in OSGi</h3>
                                                <p style="font-size: 13px">This component
is fully functional in an OSGi environment, however, it requires some actions from the user.
Hadoop uses the thread context class loader in order to load resources. Usually, the thread
context classloader will be the bundle class loader of the bundle that contains the routes.
So, the default configuration files need to be visible from the bundle class loader. A typical
way to deal with it is to keep a copy of core-default.xml in your bundle root. That file can
be found in the hadoop-common.jar.</p>
                                            </div>
                                        </div>
                                        <table id="email-actions" class="email-metadata"
cellspacing="0" cellpadding="0" border="0" width="100%" style="border-top: 1px solid #bbb;
color: #505050; margin: 8px 0 0 0; padding: 0; color: #505050">
                                            <tbody>
                                                <tr>
                                                    <td class="left" valign="top" style="font-size:
13px; padding-top: 8px; max-width: 45%; text-align: left"> <span class="email-list-item"><a
href="https://cwiki.apache.org/confluence/display/CAMEL/HDFS2" style="color: #326ca6; text-decoration:
none">View Online</a> </span> <span class="email-list-divider" style="color:
#505050; padding: 0 0.350em">&middot;</span> <span class="email-list-item"><a
href="https://cwiki.apache.org/confluence/plugins/likes/like.action?contentId=39623148" style="color:
#326ca6; text-decoration: none">Like</a> </span> </td>
                                                    <td class="right" width="50%" valign="top"
style="font-size: 13px; padding-top: 8px; text-align: right"> <span class="email-list-item"><a
href="https://cwiki.apache.org/confluence/users/removespacenotification.action?spaceKey=CAMEL"
style="color: #326ca6; text-decoration: none">Stop watching space</a> </span>
<span class="email-list-divider" style="color: #505050; padding: 0 0.350em">&middot;</span>
<span class="email-list-item"><a href="https://cwiki.apache.org/confluence/users/editmyemailsettings.action"
style="color: #326ca6; text-decoration: none">Manage Notifications</a> </span>
</td>
                                                </tr>
                                            </tbody>
                                        </table> </td>
                                </tr>
                            </tbody>
                        </table> </td>
                </tr>
                <tr>
                    <td id="email-footer" align="center" style="font-size: 13px; padding:
0 16px 32px 16px; margin: 0"> <small style="font-size: 11px"> This message was sent
by <a class="email-footer-link" style="color:#505050;font-size:11px;text-decoration:none;;
color: #326ca6; text-decoration: none; color: #505050; font-size: 11px" href="http://www.atlassian.com/software/confluence">Atlassian
Confluence</a> 5.0.3, <a class="email-footer-link" style="color:#505050;font-size:11px;text-decoration:none;;
color: #326ca6; text-decoration: none; color: #505050; font-size: 11px" href="http://www.atlassian.com/software/confluence/overview/team-collaboration-software?utm_source=email-footer">Team
Collaboration Software</a> </small> </td>
                </tr>
            </tbody>
        </table>
    </body>
</html>
Mime
View raw message