Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 6E700200D36 for ; Mon, 23 Oct 2017 07:52:43 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 6D054160BD7; Mon, 23 Oct 2017 05:52:43 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8C799160BF0 for ; Mon, 23 Oct 2017 07:52:42 +0200 (CEST) Received: (qmail 3271 invoked by uid 500); 23 Oct 2017 05:52:41 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 2163 invoked by uid 99); 23 Oct 2017 05:52:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Oct 2017 05:52:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4F857E00FF; Mon, 23 Oct 2017 05:52:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: pallavi@apache.org To: commits@falcon.apache.org Date: Mon, 23 Oct 2017 05:52:36 -0000 Message-Id: <36299dc66a144d32b9549ba937d119d1@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [02/26] falcon git commit: FALCON-2273 Disallow external entity injection and clean up some log messages archived-at: Mon, 23 Oct 2017 05:52:43 -0000 FALCON-2273 Disallow external entity injection and clean up some log messages Added to code to disable external entity resolution and not log messages with info about location of password files Author: Venkat Ranganathan Reviewers: @sandeepSamudrala, @pallavi-rao Closes #357 from vrangan/FALCON-2273 and squashes the following commits: 26e141d [Venkat Ranganathan] FALCON-2273: Disallow external entity injection and clean up some log messages 30140db [Venkat Ranganathan] Merge remote-tracking branch 'apache/master' 940be61 [Venkat Ranganathan] Merge remote-tracking branch 'apache/master' ed3a86e [Venkat Ranganathan] Merge remote-tracking branch 'apache/master' c082fa6 [Venkat Ranganathan] Merge remote-tracking branch 'apache/master' f01b8dc [Venkat Ranganathan] Merge remote-tracking branch 'apache/master' (cherry picked from commit 7628ca1edf9c440c9514b74e283d2ae77811dafe) Signed-off-by: Pallavi Rao Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7fe2c332 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7fe2c332 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7fe2c332 Branch: refs/heads/master Commit: 7fe2c33245b3032188a94f20ea83797a2843e417 Parents: 1565bde Author: Venkat Ranganathan Authored: Thu Feb 9 09:54:03 2017 +0530 Committer: Pallavi Rao Committed: Thu Feb 9 09:54:23 2017 +0530 ---------------------------------------------------------------------- .../main/java/org/apache/falcon/entity/v0/Entity.java | 6 +++++- .../java/org/apache/falcon/entity/v0/SchemaHelper.java | 12 ++++++++++++ .../java/org/apache/falcon/entity/DatasourceHelper.java | 6 ++---- .../org/apache/falcon/entity/parser/EntityParser.java | 7 ++++++- .../apache/falcon/entity/store/ConfigurationStore.java | 10 +++++++++- .../extensions/util/ExtensionProcessBuilderUtils.java | 6 +++++- 6 files changed, 39 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java ---------------------------------------------------------------------- diff --git a/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java b/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java index ba6f2e5..98faf82 100644 --- a/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java +++ b/common-types/src/main/java/org/apache/falcon/entity/v0/Entity.java @@ -20,6 +20,8 @@ package org.apache.falcon.entity.v0; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamReader; import java.io.StringReader; import java.io.StringWriter; @@ -82,7 +84,9 @@ public abstract class Entity { public static Entity fromString(EntityType type, String str) { try { Unmarshaller unmarshaler = type.getUnmarshaller(); - return (Entity) unmarshaler.unmarshal(new StringReader(str)); + XMLInputFactory xif = SchemaHelper.createXmlInputFactory(); + XMLStreamReader xsr = xif.createXMLStreamReader(new StringReader(str)); + return (Entity) unmarshaler.unmarshal(xsr); } catch (Exception e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java ---------------------------------------------------------------------- diff --git a/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java b/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java index 1c02f37..ced43fd 100644 --- a/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java +++ b/common-types/src/main/java/org/apache/falcon/entity/v0/SchemaHelper.java @@ -18,6 +18,7 @@ package org.apache.falcon.entity.v0; +import javax.xml.stream.XMLInputFactory; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -68,4 +69,15 @@ public final class SchemaHelper { throw new RuntimeException(e); } } + + /** + * Return the xml input factory that has the properties set for secure handling of data. + * @return xif + */ + public static XMLInputFactory createXmlInputFactory() { + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif; + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/common/src/main/java/org/apache/falcon/entity/DatasourceHelper.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/DatasourceHelper.java b/common/src/main/java/org/apache/falcon/entity/DatasourceHelper.java index 1479133..0178ccb 100644 --- a/common/src/main/java/org/apache/falcon/entity/DatasourceHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/DatasourceHelper.java @@ -382,13 +382,11 @@ public final class DatasourceHelper { Path path = new Path(passwordFilePath); FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri()); if (!fs.exists(path)) { - throw new IOException("The password file does not exist! " - + passwordFilePath); + throw new IOException("The password file does not exist! "); } if (!fs.isFile(path)) { - throw new IOException("The password file cannot be a directory! " - + passwordFilePath); + throw new IOException("The password file cannot be a directory! "); } InputStream is = fs.open(path); http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/common/src/main/java/org/apache/falcon/entity/parser/EntityParser.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/parser/EntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/EntityParser.java index 05b204d..f4a6372 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/EntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/EntityParser.java @@ -24,6 +24,7 @@ import org.apache.falcon.entity.store.ConfigurationStore; import org.apache.falcon.entity.v0.AccessControlList; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.security.CurrentUser; import org.apache.falcon.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -32,6 +33,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -88,9 +91,11 @@ public abstract class EntityParser { @SuppressWarnings("unchecked") public T parse(InputStream xmlStream) throws FalconException { try { + XMLInputFactory xif = SchemaHelper.createXmlInputFactory(); + XMLStreamReader xsr = xif.createXMLStreamReader(xmlStream); // parse against schema Unmarshaller unmarshaller = entityType.getUnmarshaller(); - T entity = (T) unmarshaller.unmarshal(xmlStream); + T entity = (T) unmarshaller.unmarshal(xsr); LOG.info("Parsed Entity: {}", entity.getName()); return entity; } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java index 19e10bd..11cdc05 100644 --- a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java +++ b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java @@ -24,6 +24,7 @@ import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.v0.AccessControlList; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.SchemaHelper; import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.datasource.Datasource; import org.apache.falcon.hadoop.HadoopClientFactory; @@ -41,6 +42,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -466,8 +470,12 @@ public final class ConfigurationStore implements FalconService { throws IOException, FalconException { InputStream in = fs.open(new Path(storePath, type + Path.SEPARATOR + URLEncoder.encode(name, UTF_8) + ".xml")); + XMLInputFactory xif = SchemaHelper.createXmlInputFactory(); try { - return (T) type.getUnmarshaller().unmarshal(in); + XMLStreamReader xsr = xif.createXMLStreamReader(in); + return (T) type.getUnmarshaller().unmarshal(xsr); + } catch (XMLStreamException xse) { + throw new StoreAccessException("Unable to un-marshall xml definition for " + type + "/" + name, xse); } catch (JAXBException e) { throw new StoreAccessException("Unable to un-marshall xml definition for " + type + "/" + name, e); } finally { http://git-wip-us.apache.org/repos/asf/falcon/blob/7fe2c332/extensions/src/main/java/org/apache/falcon/extensions/util/ExtensionProcessBuilderUtils.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/util/ExtensionProcessBuilderUtils.java b/extensions/src/main/java/org/apache/falcon/extensions/util/ExtensionProcessBuilderUtils.java index 286df3e..c8e870b 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/util/ExtensionProcessBuilderUtils.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/util/ExtensionProcessBuilderUtils.java @@ -39,6 +39,8 @@ import org.apache.falcon.util.NotificationType; import javax.xml.bind.Unmarshaller; import javax.xml.bind.ValidationEvent; import javax.xml.bind.ValidationEventHandler; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamReader; import java.io.StringReader; import java.util.ArrayList; import java.util.List; @@ -96,8 +98,10 @@ public final class ExtensionProcessBuilderUtils { } } ); + XMLInputFactory xif = SchemaHelper.createXmlInputFactory(); + XMLStreamReader xsr = xif.createXMLStreamReader(new StringReader(processTemplate)); process = (org.apache.falcon.entity.v0.process.Process) - unmarshaller.unmarshal(new StringReader(processTemplate)); + unmarshaller.unmarshal(xsr); } catch (Exception e) { throw new FalconException(e); }