Return-Path: X-Original-To: apmail-metron-commits-archive@minotaur.apache.org Delivered-To: apmail-metron-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5B3F618443 for ; Thu, 14 Jan 2016 17:03:31 +0000 (UTC) Received: (qmail 25171 invoked by uid 500); 14 Jan 2016 17:03:31 -0000 Delivered-To: apmail-metron-commits-archive@metron.apache.org Received: (qmail 25147 invoked by uid 500); 14 Jan 2016 17:03:31 -0000 Mailing-List: contact commits-help@metron.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@metron.incubator.apache.org Delivered-To: mailing list commits@metron.incubator.apache.org Received: (qmail 25138 invoked by uid 99); 14 Jan 2016 17:03:31 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jan 2016 17:03:31 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 95327C025E for ; Thu, 14 Jan 2016 17:03:30 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.801 X-Spam-Level: * X-Spam-Status: No, score=1.801 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=-0.001, URIBL_BLOCKED=0.001, WEIRD_PORT=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id LXApcFjm4m5v for ; Thu, 14 Jan 2016 17:03:15 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id C429A23204 for ; Thu, 14 Jan 2016 17:02:57 +0000 (UTC) Received: (qmail 22346 invoked by uid 99); 14 Jan 2016 17:02:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 14 Jan 2016 17:02:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5F12AE388C; Thu, 14 Jan 2016 17:02:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sirsean@apache.org To: commits@metron.incubator.apache.org Date: Thu, 14 Jan 2016 17:03:19 -0000 Message-Id: <2464b76b76ad4beeb6b2952019844115@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [24/85] [partial] incubator-metron git commit: Rename all OpenSOC files to Metron http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/Metron-Topologies/src/main/resources/hbase-site.xml ---------------------------------------------------------------------- diff --git a/metron-streaming/Metron-Topologies/src/main/resources/hbase-site.xml b/metron-streaming/Metron-Topologies/src/main/resources/hbase-site.xml new file mode 100644 index 0000000..8d812a9 --- /dev/null +++ b/metron-streaming/Metron-Topologies/src/main/resources/hbase-site.xml @@ -0,0 +1,131 @@ + + + + hbase.tmp.dir + /disk/h/hbase + + + hbase.hregion.memstore.chunkpool.maxsize + 0.5 + + + hbase.regionserver.codecs + lzo,gz,snappy + + + hbase.hstore.flush.retries.number + 120 + + + hbase.client.keyvalue.maxsize + 10485760 + + + hbase.rootdir + hdfs://nn1:8020/apps/hbase/data + + + hbase.defaults.for.version.skip + true + + + hbase.client.scanner.caching + 100 + + + hbase.superuser + hbase + + + hfile.block.cache.size + 0.40 + + + hbase.regionserver.checksum.verify + true + + + hbase.hregion.memstore.mslab.enabled + true + + + hbase.hregion.max.filesize + 107374182400 + + + hbase.cluster.distributed + true + + + zookeeper.session.timeout + 30000 + + + zookeeper.znode.parent + /hbase-unsecure + + + hbase.regionserver.global.memstore.lowerLimit + 0.38 + + + hbase.regionserver.handler.count + 240 + + + hbase.hregion.memstore.mslab.chunksize + 8388608 + + + hbase.zookeeper.quorum + zkpr1,zkpr2,zkpr3 + + + hbase.zookeeper.useMulti + true + + + hbase.hregion.majorcompaction + 86400000 + + + hbase.hstore.blockingStoreFiles + 200 + + + hbase.zookeeper.property.clientPort + 2181 + + + hbase.hregion.memstore.flush.size + 134217728 + + + hbase.security.authorization + false + + + hbase.regionserver.global.memstore.upperLimit + 0.4 + + + hbase.hstore.compactionThreshold + 4 + + + hbase.hregion.memstore.block.multiplier + 8 + + + hbase.security.authentication + simple + + + dfs.client.read.shortcircuit + true + + + dfs.domain.socket.path + /var/run/hdfs/dn_socket + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/pom.xml ---------------------------------------------------------------------- diff --git a/metron-streaming/pom.xml b/metron-streaming/pom.xml new file mode 100644 index 0000000..bbd4e2e --- /dev/null +++ b/metron-streaming/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + com.opensoc + OpenSOC-Streaming + 0.6BETA + pom + OpenSOC-Streaming + Stream analytics for OpenSOC + www.getopensoc.com + + @ProjectOpenSOC + 0.9.2-incubating + 0.8.0 + 2.2.0 + 0.98.0-hadoop2 + 1.1.1 + 3.0.2 + 4.4 + 18.0 + 2.2.5 + 1.7.7 + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + jsirota + James Sirota + jsirota@cisco.com + + @JamesSirota + medium.com/@JamesSirota + + + + + + + OpenSOC-Common + OpenSOC-EnrichmentAdapters + OpenSOC-MessageParsers + OpenSOC-Indexing + OpenSOC-Alerts + OpenSOC-DataLoads + OpenSOC-Topologies + OpenSOC-Pcap_Service + + + + junit + junit + 3.8.2 + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18 + + + + mode + local + + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.7 + + + false + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.3 + + 1.7 + + + + org.codehaus.mojo + emma-maven-plugin + 1.0-alpha-3 + true + + + + + + + clojars.org + http://clojars.org/repo + + + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/pom.xml.versionsBackup ---------------------------------------------------------------------- diff --git a/metron-streaming/pom.xml.versionsBackup b/metron-streaming/pom.xml.versionsBackup new file mode 100644 index 0000000..7302ae6 --- /dev/null +++ b/metron-streaming/pom.xml.versionsBackup @@ -0,0 +1,104 @@ + + + + 4.0.0 + com.opensoc + OpenSOC-Streaming + BETA_0.2 + pom + OpenSOC-Streaming + Stream analytics for OpenSOC + www.getopensoc.com + + @ProjectOpenSOC + BETA_0.2 + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + jsirota + James Sirota + jsirota@cisco.com + + @JamesSirota + medium.com/@JamesSirota + + + + + + OpenSOC-Common + OpenSOC-EnrichmentAdapters + OpenSOC-MessageParsers + OpenSOC-Indexing + OpenSOC-Alerts + OpenSOC-DataLoads + OpenSOC-Topologies + + + + junit + junit + 3.8.2 + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + mode + local + + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.7 + + + false + + + + + org.apache.maven.plugins + maven-pmd-plugin + + 1.7 + + + + org.codehaus.mojo + emma-maven-plugin + 1.0-alpha-3 + true + + + + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-streaming/readme.md ---------------------------------------------------------------------- diff --git a/metron-streaming/readme.md b/metron-streaming/readme.md new file mode 100644 index 0000000..c912153 --- /dev/null +++ b/metron-streaming/readme.md @@ -0,0 +1,13 @@ +#Current Build + +The latest build of OpenSOC-Streaming is 0.6BETA. + +We are still in the process of merging/porting additional features from our production code base into this open source release. This release will be followed by a number of additional beta releases until the port is complete. We will also work on getting additional documentation and user/developer guides to the community as soon as we can. At this time we offer no support for the beta software, but will try to respond to requests as promptly as we can. + +# OpenSOC-Streaming + +Extensible set of Storm topologies and topology attributes for streaming, enriching, indexing, and storing telemetry in Hadoop. General information on OpenSOC is available at http://opensoc.github.io + +# Documentation + +Please see documentation within each individual module for description and usage instructions. Sample topologies are provided under OpenSOC_Topologies to get you started with the framework. We pre-assume knowledge of Hadoop, Storm, Kafka, and HBase. http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/.gitignore ---------------------------------------------------------------------- diff --git a/metron-ui/.gitignore b/metron-ui/.gitignore new file mode 100644 index 0000000..c94c2a1 --- /dev/null +++ b/metron-ui/.gitignore @@ -0,0 +1,39 @@ +# Logs +logs +*.log + +# Runtime data +pids +*.pid + +# Pcap files +*.pcap + +# Config overrides +config.json + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# Deployed apps should consider commenting this line out: +# see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git +node_modules + +/.vagrant + +# Potentially sensitive seed data +/seed/es +/seed/*.pcap + +# temp files +/tmp http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/.jshintignore ---------------------------------------------------------------------- diff --git a/metron-ui/.jshintignore b/metron-ui/.jshintignore new file mode 100644 index 0000000..932c358 --- /dev/null +++ b/metron-ui/.jshintignore @@ -0,0 +1,3 @@ +lib/public +coverage +node_modules \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/.jshintrc ---------------------------------------------------------------------- diff --git a/metron-ui/.jshintrc b/metron-ui/.jshintrc new file mode 100644 index 0000000..41630dc --- /dev/null +++ b/metron-ui/.jshintrc @@ -0,0 +1,66 @@ +{ + // See http://www.jshint.com/options/ for in-depth explanations + + // Predefined globals that JSHint ignores + "browser" : true, // standard globals like 'window' + "devel" : true, // development globals, e.g. 'console' + "nonstandard" : true, // widely-adopted globals, e.g. 'escape' + "node" : true, + "jquery" : true, + + + "predef" : [ // extra globals + "angular", + "JST", + "MTD", + "google", + + // Tests + "assert", + "sinon", + "describe", + "beforeEach", + "afterEach", + "loadFixtures", + "expect", + "before", + "after", + "it", + "mixpanel", + "nv", + "d3", + + // stanford js crypto lib + "sjcl", + + // Moment JS Date library + "moment", + + // RequireJS + "requirejs", + "define", + + // Angular global obj + "angular", + + // Misc projects + "Presense", + "Refuge" + ], + + // Development + "debug" : false, // warn about debugger statements + + // Enforcing + "bitwise" : true, // prohibit the use of bitwise operations (slow and '&' is usually supposed to be '&&') + "curly" : true, // require {} for all blocks/scopes + "latedef" : true, // prohibit variable use before definition ("hoisting") + "noempty" : true, // prohibit empty blocks + "trailing" : true, // no trailing whitespace is allowed + "undef" : true, // prevent the use of undeclared variables + + // Relaxing + "sub" : true, // allow all subscript notation, including '[]' + "laxcomma" : true, // allow commas after line breaks in lists + "strict" : false // don't force strict mode +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/.nodemonignore ---------------------------------------------------------------------- diff --git a/metron-ui/.nodemonignore b/metron-ui/.nodemonignore new file mode 100644 index 0000000..fc7a8df --- /dev/null +++ b/metron-ui/.nodemonignore @@ -0,0 +1,2 @@ +.vagrant +node_modules \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/.travis.yml ---------------------------------------------------------------------- diff --git a/metron-ui/.travis.yml b/metron-ui/.travis.yml new file mode 100644 index 0000000..d496e54 --- /dev/null +++ b/metron-ui/.travis.yml @@ -0,0 +1,19 @@ +language: node_js +env: IN_TRAVIS=true +node_js: +- '0.10' +notifications: + email: + recipients: + - opensoc-github@external.cisco.com + on_success: never + on_failure: always + hipchat: + rooms: + secure: ftMVn8V34kdqbwVUDMoKgbEKG4KzywcAIxByW0bUes18Fl4e0Tc5wUajfKCtB3ih9fazNgClQgoDZWhYGZ/Ik7o/DxwJdMd7SN36Vfl412LQiV7IVPO+vvVVvrAH5RxXA1yxQveNlF7DI6ANRwVISs/OsAplznIzmyqsJ/onn1I= + +addons: + postgresql: "9.3" +before_script: + - psql -c 'create database opensoc_test;' -U postgres + - script/migrate up -e ci http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/Gruntfile.js ---------------------------------------------------------------------- diff --git a/metron-ui/Gruntfile.js b/metron-ui/Gruntfile.js new file mode 100644 index 0000000..568611f --- /dev/null +++ b/metron-ui/Gruntfile.js @@ -0,0 +1,29 @@ +module.exports = function (grunt) { + grunt.initConfig({ + // copies frontend assets from bower_components into project + // bowercopy: { + // options: { + // clean: true + // }, + // css: { + // options: { + // destPrefix: 'lib/public/css/vendor' + // }, + // files: { + // 'bootstrap.css': 'bootstrap/dist/css/bootstrap.css', + // 'bootstrap-theme.css': 'bootstrap/dist/css/bootstrap-theme.css' + // } + // }, + // libs: { + // options: { + // destPrefix: 'lib/public/js/vendor' + // }, + // files: { + // 'angular.js': 'angular/angular.js' + // } + // } + // } + }); + + grunt.loadNpmTasks('grunt-bowercopy'); +}; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/LICENSE ---------------------------------------------------------------------- diff --git a/metron-ui/LICENSE b/metron-ui/LICENSE new file mode 100644 index 0000000..ad410e1 --- /dev/null +++ b/metron-ui/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/Makefile ---------------------------------------------------------------------- diff --git a/metron-ui/Makefile b/metron-ui/Makefile new file mode 100644 index 0000000..e268401 --- /dev/null +++ b/metron-ui/Makefile @@ -0,0 +1,29 @@ +testcmd=./node_modules/istanbul/lib/cli.js cover \ + ./node_modules/mocha/bin/_mocha -- --check-leaks -R spec + +testwatchcmd=./node_modules/istanbul/lib/cli.js cover \ + ./node_modules/mocha/bin/_mocha -- --check-leaks --watch -R spec + +test: test-all + +test-watch: +ifeq ($(IN_TRAVIS),true) + PORT=4000 NODE_ENV=ci $(testwatchcmd) +else + PORT=4000 NODE_ENV=test $(testwatchcmd) +endif + +test-all: +ifeq ($(IN_TRAVIS),true) + PORT=4000 NODE_ENV=ci $(testcmd) +else + PORT=4000 NODE_ENV=test $(testcmd) +endif + +# Load test data into DB +seed: + node script/es_fetch.js && script/es_seed.sh + +clean: + rm -rf ./node_modules ./coverage + http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/README.md ---------------------------------------------------------------------- diff --git a/metron-ui/README.md b/metron-ui/README.md new file mode 100644 index 0000000..d01beb7 --- /dev/null +++ b/metron-ui/README.md @@ -0,0 +1,135 @@ +![Build Status](https://magnum.travis-ci.com/OpenSOC/opensoc-ui.svg?token=jo4ZVAV7CXvqp5459Gzo&branch=master) + +opensoc-ui +========== + +User interface for OpenSOC + +## Deployment + + Here are the minimal steps for deployment on a Ubuntu 14.04. These instructions will need to be altered for Ubuntu 12.04 as the nodejs package is too old. Assume that the code is in ```/opt/portal``` and the user is ```portal```. + +* Install dependencies: + +```bash +apt-get update +apt-get install -y libpcap-dev tshark redis-server nodejs npm +ln -s /usr/bin/nodejs /usr/bin/node +npm install -g pm2 + +su - portal +cd /opt/portal +npm install --production +``` + +* Add a file name ```config.json``` to the repo root (```/opt/portal``` in our setup). The config should point to the various services. The following is an example config, all fields are required: + +```json +{ + "secret": "some secret", + "elasticsearch": { + "url": "http://192.168.33.10:9200" + }, + "redis": { + "host": "127.0.0.1", + "port": 6379 + }, + "ldap": { + "url": "ldap://127.0.0.1:389", + "searchBase": "dc=opensoc,dc=dev", + "searchFilter": "(mail={{username}})", + "searchAttributes": ["cn", "uid", "mail", "givenName", "sn", "memberOf"], + "adminDn": "cn=admin,dc=opensoc,dc=dev", + "adminPassword": "opensoc" + }, + "permissions": { + "pcap": "cn=investigators,ou=groups,dc=opensoc,dc=dev" + } + } +``` + +* Run the server: + +```bash +pm2 start index.js -i max --name "opensoc" +``` + + +## Setup development environment + +### Step 1: Install Virtualbox and Vagrant + +Download the latest package for your platform here: + +1. [Virtualbox](https://www.virtualbox.org/wiki/Downloads) +2. [Vagrant](https://www.vagrantup.com/downloads.html) + +### Step 2: Clone repo + +```bash +git clone git@github.com:OpenSOC/opensoc-ui.git +cd opensoc-ui +``` + +### Step 3: Download and provision the development environment + +```bash +vagrant up +``` + +You might see a couple warnings, but usually these can be ignored. Check for any obvious errors as this can cause problems running the portal later. + +### Step 4: SSH into the vm +All dependencies will be installed in the VM. The repository root is shared between the host and VM. The shared volume is mounted at /vagrant. Use the following command to ssh into the newly built VM: + +```bash +vagrant ssh +cd vagrant +``` + +### Step 5: Seed the development VM + +To generate seed data for use with the opensoc-ui, use the following command. + +```bash +script/es_gen.js +``` + +On the other hand, to duplicate another ES installation use: + +```bash +ES_HOST=changeme.com script/es_fetch.js +``` + +You should now have seed data in ```seed/es```. You can load this into the dev ES instance with: + +```bash +script/es_seed +``` + +For authentication, make sure you set up the LDAP directory structure with: + +```bash +script/ldap_seed +``` + +### Step 6: Ensure tests pass + +You can now run the tests: + +```bash +make test +``` + +### Step 7: Launch the server + +The ```nodemon``` utility automatically watches for changed files and reloads the node server automatically. Run the following commands from with the vagrant vm. + +```bash +vagrant ssh +cd /vagrant +npm install -g nodemon +nodemon +``` + +You can then access the OpenSOC ui at ```http://localhost:5000```. http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/Vagrantfile ---------------------------------------------------------------------- diff --git a/metron-ui/Vagrantfile b/metron-ui/Vagrantfile new file mode 100644 index 0000000..a2e5f99 --- /dev/null +++ b/metron-ui/Vagrantfile @@ -0,0 +1,129 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + # All Vagrant configuration is done here. The most common configuration + # options are documented and commented below. For a complete reference, + # please see the online documentation at vagrantup.com. + + # Every Vagrant virtual environment requires a box to build off of. + config.vm.box = "ubuntu/trusty64" + + config.vm.provision "shell", path: 'script/provision' + + + # Nodemon server + config.vm.network :forwarded_port, guest: 5000, host: 5000 + + # Elasticsearch + config.vm.network :forwarded_port, guest: 9200, host: 9200 + + # Redis + # config.vm.network :forwarded_port, guest: 6379, host: 6379 + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # If true, then any SSH connections made will enable agent forwarding. + # Default value: false + # config.ssh.forward_agent = true + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Don't boot with headless mode + # vb.gui = true + # + # # Use VBoxManage to customize the VM. For example to change memory: + vb.customize ["modifyvm", :id, "--memory", "2048"] + end + # + # View the documentation for the provider you're using for more + # information on available options. + + # Enable provisioning with CFEngine. CFEngine Community packages are + # automatically installed. For example, configure the host as a + # policy server and optionally a policy file to run: + # + # config.vm.provision "cfengine" do |cf| + # cf.am_policy_hub = true + # # cf.run_file = "motd.cf" + # end + # + # You can also configure and bootstrap a client to an existing + # policy server: + # + # config.vm.provision "cfengine" do |cf| + # cf.policy_server_address = "10.0.2.15" + # end + + # Enable provisioning with Puppet stand alone. Puppet manifests + # are contained in a directory path relative to this Vagrantfile. + # You will need to create the manifests directory and a manifest in + # the file default.pp in the manifests_path directory. + # + # config.vm.provision "puppet" do |puppet| + # puppet.manifests_path = "manifests" + # puppet.manifest_file = "site.pp" + # end + + # Enable provisioning with chef solo, specifying a cookbooks path, roles + # path, and data_bags path (all relative to this Vagrantfile), and adding + # some recipes and/or roles. + # + # config.vm.provision "chef_solo" do |chef| + # chef.cookbooks_path = "../my-recipes/cookbooks" + # chef.roles_path = "../my-recipes/roles" + # chef.data_bags_path = "../my-recipes/data_bags" + # chef.add_recipe "mysql" + # chef.add_role "web" + # + # # You may also specify custom JSON attributes: + # chef.json = { :mysql_password => "foo" } + # end + + # Enable provisioning with chef server, specifying the chef server URL, + # and the path to the validation key (relative to this Vagrantfile). + # + # The Opscode Platform uses HTTPS. Substitute your organization for + # ORGNAME in the URL and validation key. + # + # If you have your own Chef Server, use the appropriate URL, which may be + # HTTP instead of HTTPS depending on your configuration. Also change the + # validation key to validation.pem. + # + # config.vm.provision "chef_client" do |chef| + # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" + # chef.validation_key_path = "ORGNAME-validator.pem" + # end + # + # If you're using the Opscode platform, your validator client is + # ORGNAME-validator, replacing ORGNAME with your organization name. + # + # If you have your own Chef Server, the default validation client name is + # chef-validator, unless you changed the configuration. + # + # chef.validation_client_name = "ORGNAME-validator" +end http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/bower.json ---------------------------------------------------------------------- diff --git a/metron-ui/bower.json b/metron-ui/bower.json new file mode 100644 index 0000000..f006a91 --- /dev/null +++ b/metron-ui/bower.json @@ -0,0 +1,24 @@ +{ + "name": "opensoc-ui", + "main": "index.js", + "version": "0.0.0", + "homepage": "http://opensoc.github.io/opensoc-ui/", + "authors": [ + "Jamil Bou Kheir " + ], + "description": "OpenSOC Portal", + "moduleType": [ + "node" + ], + "license": "Apache-2.0", + "private": true, + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ], + "dependencies": { + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/doc/README.md ---------------------------------------------------------------------- diff --git a/metron-ui/doc/README.md b/metron-ui/doc/README.md new file mode 100644 index 0000000..9231633 --- /dev/null +++ b/metron-ui/doc/README.md @@ -0,0 +1,4 @@ +Documentation +============= + +## Welcome to OpenSOC! http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/examples/pcap-panel/pcap-parse.html ---------------------------------------------------------------------- diff --git a/metron-ui/examples/pcap-panel/pcap-parse.html b/metron-ui/examples/pcap-panel/pcap-parse.html new file mode 100644 index 0000000..0e25dd8 --- /dev/null +++ b/metron-ui/examples/pcap-panel/pcap-parse.html @@ -0,0 +1,140 @@ + + + + + + + + + +
+
+
+
+ + + + + + + + + + +
No.TimeSourceDestinationProtocolLengthInfo
+ + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/examples/server/nginx.conf ---------------------------------------------------------------------- diff --git a/metron-ui/examples/server/nginx.conf b/metron-ui/examples/server/nginx.conf new file mode 100644 index 0000000..8700230 --- /dev/null +++ b/metron-ui/examples/server/nginx.conf @@ -0,0 +1,42 @@ +# Example OpenSOC configuration for nginx + +upstream opensoc { + server 127.0.0.1:5000; + keepalive 120; +} + +server { + client_max_body_size 1g; + + gzip on; + gzip_comp_level 6; + gzip_min_length 1000; + gzip_vary on; + gzip_proxied any; + gzip_types text/plain text/html application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; + + listen 8080; + + # Change to your host + server_name opensoc.dev; + + # Change to your web root + root /var/www/opensoc-ui/lib/public; + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + proxy_http_version 1.1; + + # Required for HTML5 Websockets (Realtime Alerts) + proxy_set_header Connection "Upgrade"; + proxy_set_header Upgrade $http_upgrade; + + proxy_pass http://opensoc; + proxy_redirect off; + break; + } +} http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/index.js ---------------------------------------------------------------------- diff --git a/metron-ui/index.js b/metron-ui/index.js new file mode 100644 index 0000000..6f8309e --- /dev/null +++ b/metron-ui/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/opensoc-ui'); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/metron-ui.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/metron-ui.js b/metron-ui/lib/metron-ui.js new file mode 100644 index 0000000..f740523 --- /dev/null +++ b/metron-ui/lib/metron-ui.js @@ -0,0 +1,102 @@ +var _ = require('lodash'); +var http = require('http'); +var path = require('path'); + +var express = require('express'); + +var connect = require('connect'); +var flash = require('connect-flash'); + +var cookieParser = require('cookie-parser'); +var bodyParser = require('body-parser'); +var cookieSession = require('cookie-session'); + +var passport = require('passport'); +var ldapauth = require('passport-ldapauth'); + +var esProxy = require('./modules/es-proxy'); +var login = require('./modules/login'); +var pcap = require('./modules/pcap'); + +var app = express(); +var config = require('./config'); + + +try { + config = _.merge(config, require('../config')); + console.log('Loaded config overrides'); +} catch(err) { + console.log('No config overrides provided'); +} + +app.set('view engine', 'jade'); +app.set('views', path.join(__dirname, 'views/')); + +// Cookie middleware +app.use(connect.logger('dev')); +app.use(flash()); +app.use(cookieParser()); +app.use(cookieSession({ + secret: config.secret, + cookie: {maxAge: 1 * 24 * 60 * 60 * 1000} // 1-day sessions +})); + +app.use(passport.initialize()); +app.use(passport.session()); + +app.use("/__es", esProxy(config)); +app.use(bodyParser.urlencoded({extended: true})); +app.use(bodyParser.json()); + +// LDAP integration +passport.use(new ldapauth.Strategy({ + usernameField: 'email', + passwordField: 'password', + server: config.ldap +}, function (user, done) { + return done(null, user); +})); + + +// Serialize LDAP user into session. +passport.serializeUser(function (ldapUser, done) { + // ensure that memberOf is an array. + var memberOf = ldapUser.memberOf || []; + memberOf = _.isArray(memberOf) ? memberOf : [memberOf]; + ldapUser.memberOf = memberOf; + + // LDAP permissions + ldapUser.permissions = {}; + var permissions = _.keys(config.permissions); + _.each(permissions, function (perm) { + var group = config.permissions[perm]; + ldapUser.permissions[perm] = _.contains(memberOf, group); + }); + + done(null, JSON.stringify(ldapUser)); +}); + + +// De-serialize user from session. +passport.deserializeUser(function (ldapUser, done) { + try { + done(null, JSON.parse(ldapUser)); + } catch(err) { + done(null, null); + } +}); + + +// Setup routes +pcap(app, config); +login(app, config); + +// Serve static assets +app.use(connect.static(path.join(__dirname, 'public'))); + + +// Start server +var server = http.createServer(app); +server.listen(config.port || 5000); + +exports.app = app; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/modules/es-proxy.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/modules/es-proxy.js b/metron-ui/lib/modules/es-proxy.js new file mode 100644 index 0000000..264b7b5 --- /dev/null +++ b/metron-ui/lib/modules/es-proxy.js @@ -0,0 +1,20 @@ +exports = module.exports = function(config) { + var httpProxy = require('http-proxy'); + var proxy = httpProxy.createProxy(); + + proxy.on('error', function (err, req, res) { + console.log("[proxyError]", err); + }); + + return function(req, res, next) { + if (!req.user) { + res.send(403, 'Forbidden!'); + return; + } + + delete req.headers.cookie; + proxy.web(req, res, { + target: config.elasticsearch.url + }); + }; +}; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/modules/login.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/modules/login.js b/metron-ui/lib/modules/login.js new file mode 100644 index 0000000..538af3e --- /dev/null +++ b/metron-ui/lib/modules/login.js @@ -0,0 +1,32 @@ +exports = module.exports = function(app, config) { + var passport = require('passport'); + + app.get('/', function (req, res, next) { + if (!req.user) { + res.redirect('/login'); + return; + } + + res.render('index', { + user: JSON.stringify(req.user), + config: JSON.stringify({ + elasticsearch: config.elasticsearch.url + }) + }); + }); + + app.get('/login', function (req, res) { + res.render('login', { flash: req.flash() }); + }); + + app.post('/login', passport.authenticate('ldapauth', { + successRedirect: '/', + failureRedirect: '/login', + failureFlash: true + })); + + app.get('/logout', function (req, res) { + req.logout(); + res.redirect('/login'); + }); +}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/modules/pcap.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/modules/pcap.js b/metron-ui/lib/modules/pcap.js new file mode 100644 index 0000000..c6091a3 --- /dev/null +++ b/metron-ui/lib/modules/pcap.js @@ -0,0 +1,95 @@ +function readRawBytes(size, transit) { + var buffer = new Buffer(size); + var bytesRead = 0; + var bytesLeft, dataLeft, len, leftOver; + var data, offset; + + while (bytesRead < size) { + if (!data || offset >= data.length) { + offset = 0; + data = transit.shift(); + } + + bytesLeft = size - bytesRead; + dataLeft = data.length - offset; + len = bytesLeft < dataLeft ? bytesLeft : dataLeft; + data.copy(buffer, bytesRead, offset, offset + len); + bytesRead += len; + offset += len; + } + + if (offset < data.length) { + dataLeft = data.length - offset; + leftOver = new Buffer(dataLeft); + data.copy(leftOver, 0, offset, offset + dataLeft); + transit.unshift(leftOver); + } + + return buffer; +} + + +exports = module.exports = function(app, config) { + var _ = require("lodash"); + var fs = require("fs"); + var spawn = require('child_process').spawn; + var querystring = require('querystring'); + var XmlStream = require('xml-stream'); + + // Mock pcap service for use in development + if (config.pcap.mock) { + app.get('/sample/pcap/:command', function(req, res) { + res.sendfile('/vagrant/seed/hbot.pcap'); + }); + } + + app.get('/pcap/:command', function(req, res) { + if (!req.user || !req.user.permissions.pcap) { + res.send(403, 'Forbidden!'); + return; + } + + var transit = []; + var pcapUrl = config.pcap.url + '/' + req.param('command'); + pcapUrl += '?' + querystring.stringify(req.query); + + var curl = spawn('curl', ['-s', pcapUrl]); + var tshark = spawn('tshark', ['-i', '-', '-T', 'pdml']); + var xml = new XmlStream(tshark.stdout); + + xml.collect('proto'); + xml.collect('field'); + + curl.stdout.pipe(tshark.stdin); + curl.stdout.on('data', function (data) { + transit.push(data); + }); + + var npcaps = 0; + xml.on('end', function() { + res.end(']}'); + curl.stdout.unpipe(tshark.stdin); + curl.kill('SIGKILL'); + tshark.kill('SIGKILL'); + }); + + xml.on('endElement: packet', function(packet) { + var psize = parseInt(packet.proto[0].$.size); + + if (!npcaps) { + res.set('Content-Type', 'application/json'); + res.write('{objects: [\n'); + // skip global header + readRawBytes(24, transit); + } else { + res.write(',\n'); + } + + // skip packet header + readRawBytes(16, transit); + packet.hexdump = readRawBytes(psize, transit).toString('hex'); + res.write(JSON.stringify(packet)); + npcaps++; + }); + }); +}; http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/app.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/app.js b/metron-ui/lib/public/app/app.js new file mode 100755 index 0000000..fa03b4a --- /dev/null +++ b/metron-ui/lib/public/app/app.js @@ -0,0 +1,158 @@ +/** + * main app level module + */ +define([ + 'angular', + 'jquery', + 'lodash', + 'require', + 'elasticjs', + 'bootstrap', + 'angular-sanitize', + 'angular-strap', + 'angular-dragdrop', + 'angular-cookies', + 'extend-jquery', + 'bindonce', +], +function (angular, $, _, appLevelRequire) { + + "use strict"; + + var app = angular.module('kibana', []), + // we will keep a reference to each module defined before boot, so that we can + // go back and allow it to define new features later. Once we boot, this will be false + pre_boot_modules = [], + // these are the functions that we need to call to register different + // features if we define them after boot time + register_fns = {}; + + // This stores the Kibana revision number, @REV@ is replaced by grunt. + app.constant('kbnVersion',"@REV@"); + + // The minimum version that must be in the cluster + app.constant('esMinVersion','0.90.9'); + + // Use this for cache busting partials + app.constant('cacheBust',"cache-bust="+Date.now()); + + /** + * Tells the application to watch the module, once bootstraping has completed + * the modules controller, service, etc. functions will be overwritten to register directly + * with this application. + * @param {[type]} module [description] + * @return {[type]} [description] + */ + app.useModule = function (module) { + if (pre_boot_modules) { + pre_boot_modules.push(module); + } else { + _.extend(module, register_fns); + } + return module; + }; + + app.safeApply = function ($scope, fn) { + switch($scope.$$phase) { + case '$apply': + // $digest hasn't started, we should be good + $scope.$eval(fn); + break; + case '$digest': + // waiting to $apply the changes + setTimeout(function () { app.safeApply($scope, fn); }, 10); + break; + default: + // clear to begin an $apply $$phase + $scope.$apply(fn); + break; + } + }; + + app.config(function ($routeProvider, $controllerProvider, $compileProvider, $filterProvider, $provide) { + + $routeProvider + .when('/dashboard', { + templateUrl: 'app/partials/dashboard.html', + }) + .when('/dashboard/:kbnType/:kbnId', { + templateUrl: 'app/partials/dashboard.html', + }) + .when('/dashboard/:kbnType/:kbnId/:params', { + templateUrl: 'app/partials/dashboard.html' + }) + .otherwise({ + redirectTo: 'dashboard' + }); + + // this is how the internet told me to dynamically add modules :/ + register_fns.controller = $controllerProvider.register; + register_fns.directive = $compileProvider.directive; + register_fns.factory = $provide.factory; + register_fns.service = $provide.service; + register_fns.filter = $filterProvider.register; + }); + + var apps_deps = [ + 'elasticjs.service', + '$strap.directives', + 'ngSanitize', + 'ngDragDrop', + 'ngCookies', + 'kibana', + 'pasvaz.bindonce' + ]; + + _.each('controllers directives factories services filters'.split(' '), + function (type) { + var module_name = 'kibana.'+type; + // create the module + app.useModule(angular.module(module_name, [])); + // push it into the apps dependencies + apps_deps.push(module_name); + }); + + app.panel_helpers = { + partial: function (name) { + return 'app/partials/'+name+'.html'; + } + }; + + // load the core components + require([ + 'controllers/all', + 'directives/all', + 'filters/all' + ], function () { + + // bootstrap the app + angular + .element(document) + .ready(function() { + $('html').attr('ng-controller', 'DashCtrl'); + angular.bootstrap(document, apps_deps) + .invoke(['$rootScope', function ($rootScope) { + _.each(pre_boot_modules, function (module) { + _.extend(module, register_fns); + }); + pre_boot_modules = false; + + $rootScope.requireContext = appLevelRequire; + $rootScope.require = function (deps, fn) { + var $scope = this; + $scope.requireContext(deps, function () { + var deps = _.toArray(arguments); + // Check that this is a valid scope. + if($scope.$id) { + $scope.$apply(function () { + fn.apply($scope, deps); + }); + } + }); + }; + }]); + }); + }); + + return app; +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/components/extend-jquery.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/components/extend-jquery.js b/metron-ui/lib/public/app/components/extend-jquery.js new file mode 100755 index 0000000..7d81aa4 --- /dev/null +++ b/metron-ui/lib/public/app/components/extend-jquery.js @@ -0,0 +1,47 @@ +define(['jquery'], +function ($) { + 'use strict'; + + /** + * jQuery extensions + */ + var $win = $(window); + + $.fn.place_tt = (function () { + var defaults = { + offset: 5, + css: { + position : 'absolute', + top : -1000, + left : 0, + color : "#c8c8c8", + padding : '10px', + 'font-size': '11pt', + 'font-weight' : 200, + 'background-color': '#1f1f1f', + 'border-radius': '5px', + 'z-index': 9999 + } + }; + + return function (x, y, opts) { + opts = $.extend(true, {}, defaults, opts); + return this.each(function () { + var $tooltip = $(this), width, height; + + $tooltip.css(opts.css); + if (!$.contains(document.body, $tooltip[0])) { + $tooltip.appendTo(document.body); + } + + width = $tooltip.outerWidth(true); + height = $tooltip.outerHeight(true); + + $tooltip.css('left', x + opts.offset + width > $win.width() ? x - opts.offset - width : x + opts.offset); + $tooltip.css('top', y + opts.offset + height > $win.height() ? y - opts.offset - height : y + opts.offset); + }); + }; + })(); + + return $; +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/components/kbn.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/components/kbn.js b/metron-ui/lib/public/app/components/kbn.js new file mode 100755 index 0000000..a42e1cb --- /dev/null +++ b/metron-ui/lib/public/app/components/kbn.js @@ -0,0 +1,632 @@ +define([ + 'jquery', + 'lodash', + 'moment', + 'chromath' +], +function($, _, moment) { + 'use strict'; + + var kbn = {}; + + kbn.get_object_fields = function(obj) { + var field_array = []; + obj = kbn.flatten_json(obj._source); + for (var field in obj) { + field_array.push(field); + } + return field_array.sort(); + }; + + kbn.get_all_fields = function(data,flat) { + return _.uniq(_.without(_.reduce(data,function(memo,hit) { + return flat ? memo.concat(_.keys(kbn.flatten_json(hit._source))) : memo.concat(_.keys(hit._source)); + },[]),'$$hashkey')); + }; + + kbn.has_field = function(obj,field) { + var obj_fields = kbn.get_object_fields(obj); + if (_.inArray(obj_fields,field) < 0) { + return false; + } else { + return true; + } + }; + + kbn.get_related_fields = function(docs,field) { + var field_array = []; + _.each(docs, function(doc) { + var keys = _.keys(doc); + if(_.contains(keys,field)) { + field_array = field_array.concat(keys); + } + }); + var counts = _.countBy(_.without(field_array,field),function(field){return field;}); + return _.map(counts, function(num, key){return {name:key,count:num};}); + }; + + kbn.recurse_field_dots = function(object,field) { + var value = null; + var nested; + if (typeof object[field] !== 'undefined') { + value = object[field]; + } + else if (nested = field.match(/(.*?)\.(.*)/)) { + if(typeof object[nested[1]] !== 'undefined') { + value = (typeof object[nested[1]][nested[2]] !== 'undefined') ? + object[nested[1]][nested[2]] : kbn.recurse_field_dots( + object[nested[1]],nested[2]); + } + } + + return value; + }; + + kbn.top_field_values = function(docs,field,count,grouped) { + var all_values = _.pluck(docs,field), + groups = {}, + counts, + hasArrays; + // manually grouping into pairs allows us to keep the original value, + _.each(all_values, function (value) { + var k; + if(_.isArray(value)) { + hasArrays = true; + } + if(_.isArray(value) && !grouped) { + k = value; + } else { + k = _.isUndefined(value) ? '' : [value.toString()]; + } + _.each(k, function(key) { + if (_.has(groups, key)) { + groups[key][1] ++; + } else { + groups[key] = [(grouped ? value : key), 1]; + } + }); + }); + + counts = _.values(groups).sort(function(a, b) { + return a[1] - b[1]; + }).reverse().slice(0,count); + + return { + counts: counts, + hasArrays : hasArrays + }; + }; + + /** + * Calculate a graph interval + * + * from:: Date object containing the start time + * to:: Date object containing the finish time + * size:: Calculate to approximately this many bars + * user_interval:: User specified histogram interval + * + */ + kbn.calculate_interval = function(from,to,size,user_interval) { + if(_.isObject(from)) { + from = from.valueOf(); + } + if(_.isObject(to)) { + to = to.valueOf(); + } + return user_interval === 0 ? kbn.round_interval((to - from)/size) : user_interval; + }; + + kbn.round_interval = function(interval) { + switch (true) { + // 0.5s + case (interval <= 500): + return 100; // 0.1s + // 5s + case (interval <= 5000): + return 1000; // 1s + // 7.5s + case (interval <= 7500): + return 5000; // 5s + // 15s + case (interval <= 15000): + return 10000; // 10s + // 45s + case (interval <= 45000): + return 30000; // 30s + // 3m + case (interval <= 180000): + return 60000; // 1m + // 9m + case (interval <= 450000): + return 300000; // 5m + // 20m + case (interval <= 1200000): + return 600000; // 10m + // 45m + case (interval <= 2700000): + return 1800000; // 30m + // 2h + case (interval <= 7200000): + return 3600000; // 1h + // 6h + case (interval <= 21600000): + return 10800000; // 3h + // 24h + case (interval <= 86400000): + return 43200000; // 12h + // 48h + case (interval <= 172800000): + return 86400000; // 24h + // 1w + case (interval <= 604800000): + return 86400000; // 24h + // 3w + case (interval <= 1814400000): + return 604800000; // 1w + // 2y + case (interval < 3628800000): + return 2592000000; // 30d + default: + return 31536000000; // 1y + } + }; + + kbn.secondsToHms = function(seconds){ + var numyears = Math.floor(seconds / 31536000); + if(numyears){ + return numyears + 'y'; + } + var numdays = Math.floor((seconds % 31536000) / 86400); + if(numdays){ + return numdays + 'd'; + } + var numhours = Math.floor(((seconds % 31536000) % 86400) / 3600); + if(numhours){ + return numhours + 'h'; + } + var numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60); + if(numminutes){ + return numminutes + 'm'; + } + var numseconds = (((seconds % 31536000) % 86400) % 3600) % 60; + if(numseconds){ + return numseconds + 's'; + } + return 'less then a second'; //'just now' //or other string you like; + }; + + kbn.to_percent = function(number,outof) { + return Math.floor((number/outof)*10000)/100 + "%"; + }; + + kbn.addslashes = function(str) { + str = str.replace(/\\/g, '\\\\'); + str = str.replace(/\'/g, '\\\''); + str = str.replace(/\"/g, '\\"'); + str = str.replace(/\0/g, '\\0'); + return str; + }; + + kbn.interval_regex = /(\d+(?:\.\d+)?)([Mwdhmsy])/; + + // histogram & trends + kbn.intervals_in_seconds = { + y: 31536000, + M: 2592000, + w: 604800, + d: 86400, + h: 3600, + m: 60, + s: 1 + }; + + kbn.describe_interval = function (string) { + var matches = string.match(kbn.interval_regex); + if (!matches || !_.has(kbn.intervals_in_seconds, matches[2])) { + throw new Error('Invalid interval string, expecting a number followed by one of "Mwdhmsy"'); + } else { + return { + sec: kbn.intervals_in_seconds[matches[2]], + type: matches[2], + count: parseFloat(matches[1]) + }; + } + }; + + kbn.interval_to_ms = function(string) { + var info = kbn.describe_interval(string); + return Math.ceil(info.sec * 1000 * info.count); + }; + + kbn.interval_to_seconds = function (string) { + var info = kbn.describe_interval(string); + return Math.ceil(info.sec * info.count); + }; + + // This should go away, moment.js can do this + kbn.time_ago = function(string) { + return new Date(new Date().getTime() - (kbn.interval_to_ms(string))); + }; + + /* This is a simplified version of elasticsearch's date parser */ + kbn.parseDate = function(text) { + if(_.isDate(text)) { + return text; + } + var time, + mathString = "", + index, + parseString; + if (text.substring(0,3) === "now") { + time = new Date(); + mathString = text.substring("now".length); + } else { + index = text.indexOf("||"); + parseString; + if (index === -1) { + parseString = text; + mathString = ""; // nothing else + } else { + parseString = text.substring(0, index); + mathString = text.substring(index + 2); + } + // We're going to just require ISO8601 timestamps, k? + time = new Date(parseString); + } + + if (!mathString.length) { + return time; + } + + //return [time,parseString,mathString]; + return kbn.parseDateMath(mathString, time); + }; + + kbn.parseDateMath = function(mathString, time, roundUp) { + var dateTime = moment(time); + for (var i = 0; i < mathString.length; ) { + var c = mathString.charAt(i++), + type, + num, + unit; + if (c === '/') { + type = 0; + } else if (c === '+') { + type = 1; + } else if (c === '-') { + type = 2; + } else { + return false; + } + + if (isNaN(mathString.charAt(i))) { + num = 1; + } else { + var numFrom = i; + while (!isNaN(mathString.charAt(i))) { + i++; + } + num = parseInt(mathString.substring(numFrom, i),10); + } + if (type === 0) { + // rounding is only allowed on whole numbers + if (num !== 1) { + return false; + } + } + unit = mathString.charAt(i++); + switch (unit) { + case 'y': + if (type === 0) { + roundUp ? dateTime.endOf('year') : dateTime.startOf('year'); + } else if (type === 1) { + dateTime.add('years',num); + } else if (type === 2) { + dateTime.subtract('years',num); + } + break; + case 'M': + if (type === 0) { + roundUp ? dateTime.endOf('month') : dateTime.startOf('month'); + } else if (type === 1) { + dateTime.add('months',num); + } else if (type === 2) { + dateTime.subtract('months',num); + } + break; + case 'w': + if (type === 0) { + roundUp ? dateTime.endOf('week') : dateTime.startOf('week'); + } else if (type === 1) { + dateTime.add('weeks',num); + } else if (type === 2) { + dateTime.subtract('weeks',num); + } + break; + case 'd': + if (type === 0) { + roundUp ? dateTime.endOf('day') : dateTime.startOf('day'); + } else if (type === 1) { + dateTime.add('days',num); + } else if (type === 2) { + dateTime.subtract('days',num); + } + break; + case 'h': + case 'H': + if (type === 0) { + roundUp ? dateTime.endOf('hour') : dateTime.startOf('hour'); + } else if (type === 1) { + dateTime.add('hours',num); + } else if (type === 2) { + dateTime.subtract('hours',num); + } + break; + case 'm': + if (type === 0) { + roundUp ? dateTime.endOf('minute') : dateTime.startOf('minute'); + } else if (type === 1) { + dateTime.add('minutes',num); + } else if (type === 2) { + dateTime.subtract('minutes',num); + } + break; + case 's': + if (type === 0) { + roundUp ? dateTime.endOf('second') : dateTime.startOf('second'); + } else if (type === 1) { + dateTime.add('seconds',num); + } else if (type === 2) { + dateTime.subtract('seconds',num); + } + break; + default: + return false; + } + } + return dateTime.toDate(); + }; + + // LOL. hahahahaha. DIE. + kbn.flatten_json = function(object,root,array) { + if (typeof array === 'undefined') { + array = {}; + } + if (typeof root === 'undefined') { + root = ''; + } + for(var index in object) { + var obj = object[index]; + var rootname = root.length === 0 ? index : root + '.' + index; + if(typeof obj === 'object' ) { + if(_.isArray(obj)) { + if(obj.length > 0 && typeof obj[0] === 'object') { + var strval = ''; + for (var objidx = 0, objlen = obj.length; objidx < objlen; objidx++) { + if (objidx > 0) { + strval = strval + ', '; + } + + strval = strval + JSON.stringify(obj[objidx]); + } + array[rootname] = strval; + } else if(obj.length === 1 && _.isNumber(obj[0])) { + array[rootname] = parseFloat(obj[0]); + } else { + array[rootname] = typeof obj === 'undefined' ? null : obj; + } + } else { + kbn.flatten_json(obj,rootname,array); + } + } else { + array[rootname] = typeof obj === 'undefined' ? null : obj; + } + } + return kbn.sortObj(array); + }; + + kbn.xmlEnt = function(value) { + if(_.isString(value)) { + var stg1 = value.replace(//g, '>') + .replace(/\r\n/g, '
') + .replace(/\r/g, '
') + .replace(/\n/g, '
') + .replace(/\t/g, '    ') + .replace(/ /g, '  ') + .replace(/<del>/g, '') + .replace(/<\/del>/g, ''); + return stg1; + } else { + return value; + } + }; + + kbn.sortObj = function(arr) { + // Setup Arrays + var sortedKeys = []; + var sortedObj = {}; + var i; + // Separate keys and sort them + for (i in arr) { + sortedKeys.push(i); + } + sortedKeys.sort(); + + // Reconstruct sorted obj based on keys + for (i in sortedKeys) { + sortedObj[sortedKeys[i]] = arr[sortedKeys[i]]; + } + return sortedObj; + }; + + kbn.query_color_dot = function (color, diameter) { + return '
'; + }; + + kbn.colorSteps = function(col,steps) { + + var _d = steps > 5 ? 1.6/steps : 0.25, // distance between steps + _p = []; // adjustment percentage + + // Create a range of numbers between -0.8 and 0.8 + for(var i = 1; i end) { + return end + 1; + } + if(start !== arr[start]) { + return start; + } + var middle = Math.floor((start + end) / 2); + + if (arr[middle] > middle) { + return kbn.smallestMissing(arr, start, middle); + } else { + return kbn.smallestMissing(arr, middle + 1, end); + } + }; + + kbn.byteFormat = function (size, decimals, min_resolution) { + var ext, steps = 0; + + if (_.isUndefined(decimals)) { + decimals = 2; + } + + if (_.isUndefined(min_resolution)) { + min_resolution = 0; + } + + + while (Math.abs(size) >= 1024) { + steps++; + size /= 1024; + min_resolution /= 1024; + } + + switch (steps) { + case 0: + ext = " B"; + break; + case 1: + ext = " KB"; + break; + case 2: + ext = " MB"; + break; + case 3: + ext = " GB"; + break; + case 4: + ext = " TB"; + break; + case 5: + ext = " PB"; + break; + case 6: + ext = " EB"; + break; + case 7: + ext = " ZB"; + break; + case 8: + ext = " YB"; + break; + } + + if (min_resolution) { + min_resolution *= Math.pow(10, decimals); + while (min_resolution % 1 !== 0) { + decimals++; + min_resolution *= 10; + } + } + + if (decimals === 0) { + decimals = undefined; + } + + return (size.toFixed(decimals) + ext); + }; + + kbn.shortFormat = function (size, decimals, min_resolution) { + var ext, steps = 0; + + if (_.isUndefined(decimals)) { + decimals = 2; + } + if (_.isUndefined(min_resolution)) { + min_resolution = 0; + } + + while (Math.abs(size) >= 1000) { + steps++; + size /= 1000; + min_resolution /= 1000; + } + + switch (steps) { + case 0: + ext = ""; + break; + case 1: + ext = " K"; + break; + case 2: + ext = " Mil"; + break; + case 3: + ext = " Bil"; + break; + case 4: + ext = " Tri"; + break; + case 5: + ext = " Quadr"; + break; + case 6: + ext = " Quint"; + break; + case 7: + ext = " Sext"; + break; + case 8: + ext = " Sept"; + break; + } + + if (min_resolution) { + min_resolution *= Math.pow(10, decimals); + while (min_resolution % 1 !== 0) { + decimals++; + min_resolution *= 10; + } + } + + if (decimals === 0) { + decimals = undefined; + } + + return (size.toFixed(decimals) + ext); + }; + + return kbn; +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/components/lodash.extended.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/components/lodash.extended.js b/metron-ui/lib/public/app/components/lodash.extended.js new file mode 100755 index 0000000..fe9c3db --- /dev/null +++ b/metron-ui/lib/public/app/components/lodash.extended.js @@ -0,0 +1,34 @@ +define([ + 'lodash-src' +], +function (_) { + 'use strict'; + + /* + Mixins :) + */ + _.mixin({ + move: function (array, fromIndex, toIndex) { + array.splice(toIndex, 0, array.splice(fromIndex, 1)[0] ); + return array; + }, + remove: function (array, index) { + array.splice(index, 1); + return array; + }, + // If variable is value, then return alt. If variable is anything else, return value; + toggle: function (variable, value, alt) { + return variable === value ? alt : value; + }, + toggleInOut: function(array,value) { + if(_.contains(array,value)) { + array = _.without(array,value); + } else { + array.push(value); + } + return array; + } + }); + + return _; +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/components/require.config.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/components/require.config.js b/metron-ui/lib/public/app/components/require.config.js new file mode 100755 index 0000000..5ea06e0 --- /dev/null +++ b/metron-ui/lib/public/app/components/require.config.js @@ -0,0 +1,98 @@ +/** + * Bootstrap require with the needed config, then load the app.js module. + */ +require.config({ + baseUrl: 'app', + // urlArgs: 'r=@REV@', + paths: { + config: '../config', + settings: 'components/settings', + kbn: 'components/kbn', + + vendor: '../vendor', + css: '../vendor/require/css', + text: '../vendor/require/text', + moment: '../vendor/moment', + blob: '../vendor/blob', + filesaver: '../vendor/filesaver', + chromath: '../vendor/chromath', + angular: '../vendor/angular/angular', + 'angular-cookies': '../vendor/angular/angular-cookies', + 'angular-dragdrop': '../vendor/angular/angular-dragdrop', + 'angular-strap': '../vendor/angular/angular-strap', + 'angular-sanitize': '../vendor/angular/angular-sanitize', + timepicker: '../vendor/angular/timepicker', + datepicker: '../vendor/angular/datepicker', + bindonce: '../vendor/angular/bindonce', + + lodash: 'components/lodash.extended', + 'lodash-src': '../vendor/lodash', + bootstrap: '../vendor/bootstrap/bootstrap', + + jquery: '../vendor/jquery/jquery-1.8.0', + 'jquery-ui': '../vendor/jquery/jquery-ui-1.10.3', + + 'extend-jquery': 'components/extend-jquery', + + 'jquery.flot': '../vendor/jquery/jquery.flot', + 'jquery.flot.pie': '../vendor/jquery/jquery.flot.pie', + 'jquery.flot.events': '../vendor/jquery/jquery.flot.events', + 'jquery.flot.selection': '../vendor/jquery/jquery.flot.selection', + 'jquery.flot.stack': '../vendor/jquery/jquery.flot.stack', + 'jquery.flot.stackpercent':'../vendor/jquery/jquery.flot.stackpercent', + 'jquery.flot.time': '../vendor/jquery/jquery.flot.time', + 'jquery.flot.byte': '../vendor/jquery/jquery.flot.byte', + + + modernizr: '../vendor/modernizr-2.6.1', + numeral: '../vendor/numeral', + elasticjs: '../vendor/elasticjs/elastic-angular-client', + }, + shim: { + angular: { + deps: ['jquery','config'], + exports: 'angular' + }, + + bootstrap: { + deps: ['jquery'] + }, + + modernizr: { + exports: 'Modernizr' + }, + + jquery: { + exports: 'jQuery' + }, + + // simple dependency declaration + // + 'jquery-ui': ['jquery'], + 'jquery.flot': ['jquery'], + 'jquery.flot.byte': ['jquery', 'jquery.flot'], + 'jquery.flot.pie': ['jquery', 'jquery.flot'], + 'jquery.flot.events': ['jquery', 'jquery.flot'], + 'jquery.flot.selection':['jquery', 'jquery.flot'], + 'jquery.flot.stack': ['jquery', 'jquery.flot'], + 'jquery.flot.stackpercent':['jquery', 'jquery.flot'], + 'jquery.flot.time': ['jquery', 'jquery.flot'], + + 'angular-sanitize': ['angular'], + 'angular-cookies': ['angular'], + 'angular-dragdrop': ['jquery','jquery-ui','angular'], + 'angular-loader': ['angular'], + 'angular-mocks': ['angular'], + 'angular-resource': ['angular'], + 'angular-route': ['angular'], + 'angular-touch': ['angular'], + 'bindonce': ['angular'], + 'angular-strap': ['angular', 'bootstrap','timepicker', 'datepicker'], + + timepicker: ['jquery', 'bootstrap'], + datepicker: ['jquery', 'bootstrap'], + + elasticjs: ['angular', '../vendor/elasticjs/elastic'] + }, + waitSeconds: 60, +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/components/settings.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/components/settings.js b/metron-ui/lib/public/app/components/settings.js new file mode 100755 index 0000000..6416286 --- /dev/null +++ b/metron-ui/lib/public/app/components/settings.js @@ -0,0 +1,28 @@ +define(['lodash'], +function (_) { + "use strict"; + + return function Settings (options) { + /** + * To add a setting, you MUST define a default. Also, + * THESE ARE ONLY DEFAULTS. + * They are overridden by config.js in the root directory + * @type {Object} + */ + var defaults = { + elasticsearch : "http://"+window.location.hostname+":9200", + panel_names : [], + kibana_index : 'kibana-int', + default_route : '/dashboard/file/default.json' + }; + + // This initializes a new hash on purpose, to avoid adding parameters to + // config.js without providing sane defaults + var settings = {}; + _.each(defaults, function(value, key) { + settings[key] = typeof options[key] !== 'undefined' ? options[key] : defaults[key]; + }); + + return settings; + }; +}); http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0648a447/metron-ui/lib/public/app/controllers/all.js ---------------------------------------------------------------------- diff --git a/metron-ui/lib/public/app/controllers/all.js b/metron-ui/lib/public/app/controllers/all.js new file mode 100755 index 0000000..05813c0 --- /dev/null +++ b/metron-ui/lib/public/app/controllers/all.js @@ -0,0 +1,6 @@ +define([ + './dash', + './dashLoader', + './row', + './pulldown' +], function () {});