From commits-return-7172-archive-asf-public=cust-asf.ponee.io@zookeeper.apache.org Wed Oct 10 12:30:14 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 4C22A1807DA for ; Wed, 10 Oct 2018 12:30:08 +0200 (CEST) Received: (qmail 33992 invoked by uid 500); 10 Oct 2018 10:30:03 -0000 Mailing-List: contact commits-help@zookeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zookeeper.apache.org Delivered-To: mailing list commits@zookeeper.apache.org Received: (qmail 31697 invoked by uid 99); 10 Oct 2018 10:30:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 10 Oct 2018 10:30:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D090AE11BD; Wed, 10 Oct 2018 10:30:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: andor@apache.org To: commits@zookeeper.apache.org Date: Wed, 10 Oct 2018 10:30:35 -0000 Message-Id: <61f0d697d85a4c00a48cdac0288c02b8@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [36/51] [partial] zookeeper git commit: ZOOKEEPER-3032: MAVEN MIGRATION - branch 3.4 move java server, client http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/ZooTrace.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/ZooTrace.java b/src/java/main/org/apache/zookeeper/server/ZooTrace.java deleted file mode 100644 index 946a4bf..0000000 --- a/src/java/main/org/apache/zookeeper/server/ZooTrace.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.zookeeper.server.quorum.LearnerHandler; -import org.apache.zookeeper.server.quorum.QuorumPacket; - -/** - * This class encapsulates and centralizes tracing for the ZooKeeper server. - * Trace messages go to the log with TRACE level. - *

- * Log4j must be correctly configured to capture the TRACE messages. - */ -public class ZooTrace { - final static public long CLIENT_REQUEST_TRACE_MASK = 1 << 1; - - final static public long CLIENT_DATA_PACKET_TRACE_MASK = 1 << 2; - - final static public long CLIENT_PING_TRACE_MASK = 1 << 3; - - final static public long SERVER_PACKET_TRACE_MASK = 1 << 4; - - final static public long SESSION_TRACE_MASK = 1 << 5; - - final static public long EVENT_DELIVERY_TRACE_MASK = 1 << 6; - - final static public long SERVER_PING_TRACE_MASK = 1 << 7; - - final static public long WARNING_TRACE_MASK = 1 << 8; - - final static public long JMX_TRACE_MASK = 1 << 9; - - private static long traceMask = CLIENT_REQUEST_TRACE_MASK - | SERVER_PACKET_TRACE_MASK | SESSION_TRACE_MASK - | WARNING_TRACE_MASK; - - public static long getTextTraceLevel() { - return traceMask; - } - - public static void setTextTraceLevel(long mask) { - traceMask = mask; - Logger LOG = LoggerFactory.getLogger(ZooTrace.class); - LOG.info("Set text trace mask to 0x" + Long.toHexString(mask)); - } - - public static boolean isTraceEnabled(Logger log, long mask) { - return log.isTraceEnabled() && (mask & traceMask) != 0; - } - - public static void logTraceMessage(Logger log, long mask, String msg) { - if (isTraceEnabled(log, mask)) { - log.trace(msg); - } - } - - static public void logQuorumPacket(Logger log, long mask, - char direction, QuorumPacket qp) - { - if (isTraceEnabled(log, mask)) { - logTraceMessage(log, mask, direction + - " " + LearnerHandler.packetToString(qp)); - } - } - - static public void logRequest(Logger log, long mask, - char rp, Request request, String header) - { - if (isTraceEnabled(log, mask)) { - log.trace(header + ":" + rp + request.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/AuthenticationProvider.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/AuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/AuthenticationProvider.java deleted file mode 100644 index dd6e32f..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/AuthenticationProvider.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.server.ServerCnxn; - -/** - * This interface is implemented by authentication providers to add new kinds of - * authentication schemes to ZooKeeper. - */ -public interface AuthenticationProvider { - /** - * The String used to represent this provider. This will correspond to the - * scheme field of an Id. - * - * @return the scheme of this provider. - */ - String getScheme(); - - /** - * This method is called when a client passes authentication data for this - * scheme. The authData is directly from the authentication packet. The - * implementor may attach new ids to the authInfo field of cnxn or may use - * cnxn to send packets back to the client. - * - * @param cnxn - * the cnxn that received the authentication information. - * @param authData - * the authentication data received. - * @return TODO - */ - KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte authData[]); - - /** - * This method is called to see if the given id matches the given id - * expression in the ACL. This allows schemes to use application specific - * wild cards. - * - * @param id - * the id to check. - * @param aclExpr - * the expression to match ids against. - * @return true if the id can be matched by the expression. - */ - boolean matches(String id, String aclExpr); - - /** - * This method is used to check if the authentication done by this provider - * should be used to identify the creator of a node. Some ids such as hosts - * and ip addresses are rather transient and in general don't really - * identify a client even though sometimes they do. - * - * @return true if this provider identifies creators. - */ - boolean isAuthenticated(); - - /** - * Validates the syntax of an id. - * - * @param id - * the id to validate. - * @return true if id is well formed. - */ - boolean isValid(String id); -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/DigestAuthenticationProvider.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/DigestAuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/DigestAuthenticationProvider.java deleted file mode 100644 index 2427e7a..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/DigestAuthenticationProvider.java +++ /dev/null @@ -1,139 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; - -public class DigestAuthenticationProvider implements AuthenticationProvider { - private static final Logger LOG = - LoggerFactory.getLogger(DigestAuthenticationProvider.class); - - /** specify a command line property with key of - * "zookeeper.DigestAuthenticationProvider.superDigest" - * and value of "super:" to enable - * super user access (i.e. acls disabled) - */ - private final static String superDigest = System.getProperty( - "zookeeper.DigestAuthenticationProvider.superDigest"); - - public String getScheme() { - return "digest"; - } - - static final private String base64Encode(byte b[]) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < b.length;) { - int pad = 0; - int v = (b[i++] & 0xff) << 16; - if (i < b.length) { - v |= (b[i++] & 0xff) << 8; - } else { - pad++; - } - if (i < b.length) { - v |= (b[i++] & 0xff); - } else { - pad++; - } - sb.append(encode(v >> 18)); - sb.append(encode(v >> 12)); - if (pad < 2) { - sb.append(encode(v >> 6)); - } else { - sb.append('='); - } - if (pad < 1) { - sb.append(encode(v)); - } else { - sb.append('='); - } - } - return sb.toString(); - } - - static final private char encode(int i) { - i &= 0x3f; - if (i < 26) { - return (char) ('A' + i); - } - if (i < 52) { - return (char) ('a' + i - 26); - } - if (i < 62) { - return (char) ('0' + i - 52); - } - return i == 62 ? '+' : '/'; - } - - static public String generateDigest(String idPassword) - throws NoSuchAlgorithmException { - String parts[] = idPassword.split(":", 2); - byte digest[] = MessageDigest.getInstance("SHA1").digest( - idPassword.getBytes()); - return parts[0] + ":" + base64Encode(digest); - } - - public KeeperException.Code - handleAuthentication(ServerCnxn cnxn, byte[] authData) - { - String id = new String(authData); - try { - String digest = generateDigest(id); - if (digest.equals(superDigest)) { - cnxn.addAuthInfo(new Id("super", "")); - } - cnxn.addAuthInfo(new Id(getScheme(), digest)); - return KeeperException.Code.OK; - } catch (NoSuchAlgorithmException e) { - LOG.error("Missing algorithm",e); - } - return KeeperException.Code.AUTHFAILED; - } - - public boolean isAuthenticated() { - return true; - } - - public boolean isValid(String id) { - String parts[] = id.split(":"); - return parts.length == 2; - } - - public boolean matches(String id, String aclExpr) { - return id.equals(aclExpr); - } - - /** Call with a single argument of user:pass to generate authdata. - * Authdata output can be used when setting superDigest for example. - * @param args single argument of user:pass - * @throws NoSuchAlgorithmException - */ - public static void main(String args[]) throws NoSuchAlgorithmException { - for (int i = 0; i < args.length; i++) { - System.out.println(args[i] + "->" + generateDigest(args[i])); - } - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/DigestLoginModule.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/DigestLoginModule.java b/src/java/main/org/apache/zookeeper/server/auth/DigestLoginModule.java deleted file mode 100644 index 955c728..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/DigestLoginModule.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import javax.security.auth.Subject; -import javax.security.auth.callback.CallbackHandler; -import javax.security.auth.spi.LoginModule; -import java.util.Map; - -public class DigestLoginModule implements LoginModule { - private Subject subject; - - public boolean abort() { - return false; - } - - public boolean commit() { - return true; - } - - public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { - if (options.containsKey("username")) { - // Zookeeper client: get username and password from JAAS conf (only used if using DIGEST-MD5). - this.subject = subject; - String username = (String)options.get("username"); - this.subject.getPublicCredentials().add((Object)username); - String password = (String)options.get("password"); - this.subject.getPrivateCredentials().add((Object)password); - } - return; - } - - public boolean logout() { - return true; - } - - public boolean login() { - // Unlike with Krb5LoginModule, we don't do any actual login or credential passing here: authentication to Zookeeper - // is done later, through the SASLClient object. - return true; - } - -} - - http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/IPAuthenticationProvider.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/IPAuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/IPAuthenticationProvider.java deleted file mode 100644 index 192dc06..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/IPAuthenticationProvider.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.server.ServerCnxn; - -public class IPAuthenticationProvider implements AuthenticationProvider { - - public String getScheme() { - return "ip"; - } - - public KeeperException.Code - handleAuthentication(ServerCnxn cnxn, byte[] authData) - { - String id = cnxn.getRemoteSocketAddress().getAddress().getHostAddress(); - cnxn.addAuthInfo(new Id(getScheme(), id)); - return KeeperException.Code.OK; - } - - // This is a bit weird but we need to return the address and the number of - // bytes (to distinguish between IPv4 and IPv6 - private byte[] addr2Bytes(String addr) { - byte b[] = v4addr2Bytes(addr); - // TODO Write the v6addr2Bytes - return b; - } - - private byte[] v4addr2Bytes(String addr) { - String parts[] = addr.split("\\.", -1); - if (parts.length != 4) { - return null; - } - byte b[] = new byte[4]; - for (int i = 0; i < 4; i++) { - try { - int v = Integer.parseInt(parts[i]); - if (v >= 0 && v <= 255) { - b[i] = (byte) v; - } else { - return null; - } - } catch (NumberFormatException e) { - return null; - } - } - return b; - } - - private void mask(byte b[], int bits) { - int start = bits / 8; - int startMask = (1 << (8 - (bits % 8))) - 1; - startMask = ~startMask; - while (start < b.length) { - b[start] &= startMask; - startMask = 0; - start++; - } - } - - public boolean matches(String id, String aclExpr) { - String parts[] = aclExpr.split("/", 2); - byte aclAddr[] = addr2Bytes(parts[0]); - if (aclAddr == null) { - return false; - } - int bits = aclAddr.length * 8; - if (parts.length == 2) { - try { - bits = Integer.parseInt(parts[1]); - if (bits < 0 || bits > aclAddr.length * 8) { - return false; - } - } catch (NumberFormatException e) { - return false; - } - } - mask(aclAddr, bits); - byte remoteAddr[] = addr2Bytes(id); - if (remoteAddr == null) { - return false; - } - mask(remoteAddr, bits); - for (int i = 0; i < remoteAddr.length; i++) { - if (remoteAddr[i] != aclAddr[i]) { - return false; - } - } - return true; - } - - public boolean isAuthenticated() { - return false; - } - - public boolean isValid(String id) { - return addr2Bytes(id) != null; - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/KerberosName.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/KerberosName.java b/src/java/main/org/apache/zookeeper/server/auth/KerberosName.java deleted file mode 100644 index 01d1f66..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/KerberosName.java +++ /dev/null @@ -1,418 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - /* This file copied from Hadoop's security branch, - * with the following changes: - * 1. package changed from org.apache.hadoop.security to - * org.apache.zookeeper.server.auth. - * 2. Usage of Hadoop's Configuration class removed since - * it is not available in Zookeeper: instead, system property - * "zookeeper.security.auth_to_local" is used. - */ - -package org.apache.zookeeper.server.auth; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.zookeeper.server.util.KerberosUtil; - -/** - * This class implements parsing and handling of Kerberos principal names. In - * particular, it splits them apart and translates them down into local - * operating system names. - */ -public class KerberosName { - /** The first component of the name */ - private final String serviceName; - /** The second component of the name. It may be null. */ - private final String hostName; - /** The realm of the name. */ - private final String realm; - - /** - * A pattern that matches a Kerberos name with at most 2 components. - */ - private static final Pattern nameParser = - Pattern.compile("([^/@]*)(/([^/@]*))?@([^/@]*)"); - - /** - * A pattern that matches a string with out '$' and then a single - * parameter with $n. - */ - private static Pattern parameterPattern = - Pattern.compile("([^$]*)(\\$(\\d*))?"); - - /** - * A pattern for parsing a auth_to_local rule. - */ - private static final Pattern ruleParser = - Pattern.compile("\\s*((DEFAULT)|(RULE:\\[(\\d*):([^\\]]*)](\\(([^)]*)\\))?"+ - "(s/([^/]*)/([^/]*)/(g)?)?))"); - - /** - * A pattern that recognizes simple/non-simple names. - */ - private static final Pattern nonSimplePattern = Pattern.compile("[/@]"); - - /** - * The list of translation rules. - */ - private static List rules; - - private static String defaultRealm; - - static { - try { - defaultRealm = KerberosUtil.getDefaultRealm(); - } catch (Exception ke) { - if ((System.getProperty("zookeeper.requireKerberosConfig") != null) && - (System.getProperty("zookeeper.requireKerberosConfig").equals("true"))) { - throw new IllegalArgumentException("Can't get Kerberos configuration",ke); - } - else - defaultRealm=""; - } - try { - // setConfiguration() will work even if the above try() fails due - // to a missing Kerberos configuration (unless zookeeper.requireKerberosConfig - // is set to true, which would not allow execution to reach here due to the - // throwing of an IllegalArgumentException above). - setConfiguration(); - } - catch (IOException e) { - throw new IllegalArgumentException("Could not configure Kerberos principal name mapping."); - } - } - - /** - * Create a name from the full Kerberos principal name. - * @param name - */ - public KerberosName(String name) { - Matcher match = nameParser.matcher(name); - if (!match.matches()) { - if (name.contains("@")) { - throw new IllegalArgumentException("Malformed Kerberos name: " + name); - } else { - serviceName = name; - hostName = null; - realm = null; - } - } else { - serviceName = match.group(1); - hostName = match.group(3); - realm = match.group(4); - } - } - - /** - * Get the configured default realm. - * @return the default realm from the krb5.conf - */ - public String getDefaultRealm() { - return defaultRealm; - } - - /** - * Put the name back together from the parts. - */ - @Override - public String toString() { - StringBuilder result = new StringBuilder(); - result.append(serviceName); - if (hostName != null) { - result.append('/'); - result.append(hostName); - } - if (realm != null) { - result.append('@'); - result.append(realm); - } - return result.toString(); - } - - /** - * Get the first component of the name. - * @return the first section of the Kerberos principal name - */ - public String getServiceName() { - return serviceName; - } - - /** - * Get the second component of the name. - * @return the second section of the Kerberos principal name, and may be null - */ - public String getHostName() { - return hostName; - } - - /** - * Get the realm of the name. - * @return the realm of the name, may be null - */ - public String getRealm() { - return realm; - } - - /** - * An encoding of a rule for translating kerberos names. - */ - private static class Rule { - private final boolean isDefault; - private final int numOfComponents; - private final String format; - private final Pattern match; - private final Pattern fromPattern; - private final String toPattern; - private final boolean repeat; - - Rule() { - isDefault = true; - numOfComponents = 0; - format = null; - match = null; - fromPattern = null; - toPattern = null; - repeat = false; - } - - Rule(int numOfComponents, String format, String match, String fromPattern, - String toPattern, boolean repeat) { - isDefault = false; - this.numOfComponents = numOfComponents; - this.format = format; - this.match = match == null ? null : Pattern.compile(match); - this.fromPattern = - fromPattern == null ? null : Pattern.compile(fromPattern); - this.toPattern = toPattern; - this.repeat = repeat; - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder(); - if (isDefault) { - buf.append("DEFAULT"); - } else { - buf.append("RULE:["); - buf.append(numOfComponents); - buf.append(':'); - buf.append(format); - buf.append(']'); - if (match != null) { - buf.append('('); - buf.append(match); - buf.append(')'); - } - if (fromPattern != null) { - buf.append("s/"); - buf.append(fromPattern); - buf.append('/'); - buf.append(toPattern); - buf.append('/'); - if (repeat) { - buf.append('g'); - } - } - } - return buf.toString(); - } - - /** - * Replace the numbered parameters of the form $n where n is from 1 to - * the length of params. Normal text is copied directly and $n is replaced - * by the corresponding parameter. - * @param format the string to replace parameters again - * @param params the list of parameters - * @return the generated string with the parameter references replaced. - * @throws BadFormatString - */ - static String replaceParameters(String format, - String[] params) throws BadFormatString { - Matcher match = parameterPattern.matcher(format); - int start = 0; - StringBuilder result = new StringBuilder(); - while (start < format.length() && match.find(start)) { - result.append(match.group(1)); - String paramNum = match.group(3); - if (paramNum != null) { - try { - int num = Integer.parseInt(paramNum); - if (num < 0 || num > params.length) { - throw new BadFormatString("index " + num + " from " + format + - " is outside of the valid range 0 to " + - (params.length - 1)); - } - result.append(params[num]); - } catch (NumberFormatException nfe) { - throw new BadFormatString("bad format in username mapping in " + - paramNum, nfe); - } - - } - start = match.end(); - } - return result.toString(); - } - - /** - * Replace the matches of the from pattern in the base string with the value - * of the to string. - * @param base the string to transform - * @param from the pattern to look for in the base string - * @param to the string to replace matches of the pattern with - * @param repeat whether the substitution should be repeated - * @return - */ - static String replaceSubstitution(String base, Pattern from, String to, - boolean repeat) { - Matcher match = from.matcher(base); - if (repeat) { - return match.replaceAll(to); - } else { - return match.replaceFirst(to); - } - } - - /** - * Try to apply this rule to the given name represented as a parameter - * array. - * @param params first element is the realm, second and later elements are - * are the components of the name "a/b@FOO" -> {"FOO", "a", "b"} - * @return the short name if this rule applies or null - * @throws IOException throws if something is wrong with the rules - */ - String apply(String[] params) throws IOException { - String result = null; - if (isDefault) { - if (defaultRealm.equals(params[0])) { - result = params[1]; - } - } else if (params.length - 1 == numOfComponents) { - String base = replaceParameters(format, params); - if (match == null || match.matcher(base).matches()) { - if (fromPattern == null) { - result = base; - } else { - result = replaceSubstitution(base, fromPattern, toPattern, repeat); - } - } - } - if (result != null && nonSimplePattern.matcher(result).find()) { - throw new NoMatchingRule("Non-simple name " + result + - " after auth_to_local rule " + this); - } - return result; - } - } - - static List parseRules(String rules) { - List result = new ArrayList(); - String remaining = rules.trim(); - while (remaining.length() > 0) { - Matcher matcher = ruleParser.matcher(remaining); - if (!matcher.lookingAt()) { - throw new IllegalArgumentException("Invalid rule: " + remaining); - } - if (matcher.group(2) != null) { - result.add(new Rule()); - } else { - result.add(new Rule(Integer.parseInt(matcher.group(4)), - matcher.group(5), - matcher.group(7), - matcher.group(9), - matcher.group(10), - "g".equals(matcher.group(11)))); - } - remaining = remaining.substring(matcher.end()); - } - return result; - } - - /** - * Set the static configuration to get the rules. - * @param conf the new configuration - * @throws IOException - */ - public static void setConfiguration() throws IOException { - String ruleString = System.getProperty("zookeeper.security.auth_to_local", "DEFAULT"); - rules = parseRules(ruleString); - } - - @SuppressWarnings("serial") - public static class BadFormatString extends IOException { - BadFormatString(String msg) { - super(msg); - } - BadFormatString(String msg, Throwable err) { - super(msg, err); - } - } - - @SuppressWarnings("serial") - public static class NoMatchingRule extends IOException { - NoMatchingRule(String msg) { - super(msg); - } - } - - /** - * Get the translation of the principal name into an operating system - * user name. - * @return the short name - * @throws IOException - */ - public String getShortName() throws IOException { - String[] params; - if (hostName == null) { - // if it is already simple, just return it - if (realm == null) { - return serviceName; - } - params = new String[]{realm, serviceName}; - } else { - params = new String[]{realm, serviceName, hostName}; - } - for(Rule r: rules) { - String result = r.apply(params); - if (result != null) { - return result; - } - } - throw new NoMatchingRule("No rules applied to " + toString()); - } - - static void printRules() throws IOException { - int i = 0; - for(Rule r: rules) { - System.out.println(++i + " " + r); - } - } - - public static void main(String[] args) throws Exception { - for(String arg: args) { - KerberosName name = new KerberosName(arg); - System.out.println("Name: " + name + " to " + name.getShortName()); - } - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/ProviderRegistry.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/ProviderRegistry.java b/src/java/main/org/apache/zookeeper/server/auth/ProviderRegistry.java deleted file mode 100644 index 855971b..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/ProviderRegistry.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import java.util.Enumeration; -import java.util.HashMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.zookeeper.server.ZooKeeperServer; - -public class ProviderRegistry { - private static final Logger LOG = LoggerFactory.getLogger(ProviderRegistry.class); - - private static boolean initialized = false; - private static HashMap authenticationProviders = - new HashMap(); - - public static void initialize() { - synchronized (ProviderRegistry.class) { - if (initialized) - return; - IPAuthenticationProvider ipp = new IPAuthenticationProvider(); - DigestAuthenticationProvider digp = new DigestAuthenticationProvider(); - authenticationProviders.put(ipp.getScheme(), ipp); - authenticationProviders.put(digp.getScheme(), digp); - Enumeration en = System.getProperties().keys(); - while (en.hasMoreElements()) { - String k = (String) en.nextElement(); - if (k.startsWith("zookeeper.authProvider.")) { - String className = System.getProperty(k); - try { - Class c = ZooKeeperServer.class.getClassLoader() - .loadClass(className); - AuthenticationProvider ap = (AuthenticationProvider) c - .getDeclaredConstructor().newInstance(); - authenticationProviders.put(ap.getScheme(), ap); - } catch (Exception e) { - LOG.warn("Problems loading " + className,e); - } - } - } - initialized = true; - } - } - - public static AuthenticationProvider getProvider(String scheme) { - if(!initialized) - initialize(); - return authenticationProviders.get(scheme); - } - - public static String listProviders() { - StringBuilder sb = new StringBuilder(); - for(String s: authenticationProviders.keySet()) { - sb.append(s + " "); -} - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/SASLAuthenticationProvider.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/SASLAuthenticationProvider.java b/src/java/main/org/apache/zookeeper/server/auth/SASLAuthenticationProvider.java deleted file mode 100644 index 063304f..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/SASLAuthenticationProvider.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.server.ServerCnxn; - -public class SASLAuthenticationProvider implements AuthenticationProvider { - - public String getScheme() { - return "sasl"; - } - - public KeeperException.Code - handleAuthentication(ServerCnxn cnxn, byte[] authData) - { - // Should never call this: SASL authentication is negotiated at session initiation. - // TODO: consider substituting current implementation of direct ClientCnxn manipulation with - // a call to this method (SASLAuthenticationProvider:handleAuthentication()) at session initiation. - return KeeperException.Code.AUTHFAILED; - - } - - public boolean matches(String id,String aclExpr) { - if (System.getProperty("zookeeper.superUser") != null) { - return (id.equals(System.getProperty("zookeeper.superUser")) || id.equals(aclExpr)); - } - return (id.equals("super") || id.equals(aclExpr)); - } - - public boolean isAuthenticated() { - return true; - } - - public boolean isValid(String id) { - // Since the SASL authenticator will usually be used with Kerberos authentication, - // it should enforce that these names are valid according to Kerberos's - // syntax for principals. - // - // Use the KerberosName(id) constructor to define validity: - // if KerberosName(id) throws IllegalArgumentException, then id is invalid. - // otherwise, it is valid. - // - try { - new KerberosName(id); - return true; - } - catch (IllegalArgumentException e) { - return false; - } - } - - -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/auth/SaslServerCallbackHandler.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/auth/SaslServerCallbackHandler.java b/src/java/main/org/apache/zookeeper/server/auth/SaslServerCallbackHandler.java deleted file mode 100644 index 9f53a4d..0000000 --- a/src/java/main/org/apache/zookeeper/server/auth/SaslServerCallbackHandler.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.auth; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; -import javax.security.auth.callback.NameCallback; -import javax.security.auth.callback.PasswordCallback; -import javax.security.auth.callback.UnsupportedCallbackException; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; -import javax.security.sasl.AuthorizeCallback; -import javax.security.sasl.RealmCallback; - -import org.apache.zookeeper.server.ZooKeeperSaslServer; - -public class SaslServerCallbackHandler implements CallbackHandler { - private static final String USER_PREFIX = "user_"; - private static final Logger LOG = LoggerFactory.getLogger(SaslServerCallbackHandler.class); - private static final String SYSPROP_SUPER_PASSWORD = "zookeeper.SASLAuthenticationProvider.superPassword"; - private static final String SYSPROP_REMOVE_HOST = "zookeeper.kerberos.removeHostFromPrincipal"; - private static final String SYSPROP_REMOVE_REALM = "zookeeper.kerberos.removeRealmFromPrincipal"; - - private String userName; - private final Map credentials = new HashMap(); - - public SaslServerCallbackHandler(Configuration configuration) - throws IOException { - String serverSection = System.getProperty( - ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY, - ZooKeeperSaslServer.DEFAULT_LOGIN_CONTEXT_NAME); - AppConfigurationEntry configurationEntries[] = configuration.getAppConfigurationEntry(serverSection); - - if (configurationEntries == null) { - String errorMessage = "Could not find a '" + serverSection + "' entry in this configuration: Server cannot start."; - LOG.error(errorMessage); - throw new IOException(errorMessage); - } - credentials.clear(); - for(AppConfigurationEntry entry: configurationEntries) { - Map options = entry.getOptions(); - // Populate DIGEST-MD5 user -> password map with JAAS configuration entries from the "Server" section. - // Usernames are distinguished from other options by prefixing the username with a "user_" prefix. - for(Map.Entry pair : options.entrySet()) { - String key = pair.getKey(); - if (key.startsWith(USER_PREFIX)) { - String userName = key.substring(USER_PREFIX.length()); - credentials.put(userName,(String)pair.getValue()); - } - } - } - } - - public void handle(Callback[] callbacks) throws UnsupportedCallbackException { - for (Callback callback : callbacks) { - if (callback instanceof NameCallback) { - handleNameCallback((NameCallback) callback); - } else if (callback instanceof PasswordCallback) { - handlePasswordCallback((PasswordCallback) callback); - } else if (callback instanceof RealmCallback) { - handleRealmCallback((RealmCallback) callback); - } else if (callback instanceof AuthorizeCallback) { - handleAuthorizeCallback((AuthorizeCallback) callback); - } - } - } - - private void handleNameCallback(NameCallback nc) { - // check to see if this user is in the user password database. - if (credentials.get(nc.getDefaultName()) == null) { - LOG.warn("User '" + nc.getDefaultName() + "' not found in list of DIGEST-MD5 authenticateable users."); - return; - } - nc.setName(nc.getDefaultName()); - userName = nc.getDefaultName(); - } - - private void handlePasswordCallback(PasswordCallback pc) { - if ("super".equals(this.userName) && System.getProperty(SYSPROP_SUPER_PASSWORD) != null) { - // superuser: use Java system property for password, if available. - pc.setPassword(System.getProperty(SYSPROP_SUPER_PASSWORD).toCharArray()); - } else if (credentials.containsKey(userName) ) { - pc.setPassword(credentials.get(userName).toCharArray()); - } else { - LOG.warn("No password found for user: " + userName); - } - } - - private void handleRealmCallback(RealmCallback rc) { - LOG.debug("client supplied realm: " + rc.getDefaultText()); - rc.setText(rc.getDefaultText()); - } - - private void handleAuthorizeCallback(AuthorizeCallback ac) { - String authenticationID = ac.getAuthenticationID(); - String authorizationID = ac.getAuthorizationID(); - - LOG.info("Successfully authenticated client: authenticationID=" + authenticationID - + "; authorizationID=" + authorizationID + "."); - ac.setAuthorized(true); - - // canonicalize authorization id according to system properties: - // zookeeper.kerberos.removeRealmFromPrincipal(={true,false}) - // zookeeper.kerberos.removeHostFromPrincipal(={true,false}) - KerberosName kerberosName = new KerberosName(authenticationID); - try { - StringBuilder userNameBuilder = new StringBuilder(kerberosName.getShortName()); - if (shouldAppendHost(kerberosName)) { - userNameBuilder.append("/").append(kerberosName.getHostName()); - } - if (shouldAppendRealm(kerberosName)) { - userNameBuilder.append("@").append(kerberosName.getRealm()); - } - LOG.info("Setting authorizedID: " + userNameBuilder); - ac.setAuthorizedID(userNameBuilder.toString()); - } catch (IOException e) { - LOG.error("Failed to set name based on Kerberos authentication rules.", e); - } - } - - private boolean shouldAppendRealm(KerberosName kerberosName) { - return !isSystemPropertyTrue(SYSPROP_REMOVE_REALM) && kerberosName.getRealm() != null; - } - - private boolean shouldAppendHost(KerberosName kerberosName) { - return !isSystemPropertyTrue(SYSPROP_REMOVE_HOST) && kerberosName.getHostName() != null; - } - - private boolean isSystemPropertyTrue(String propertyName) { - return "true".equals(System.getProperty(propertyName)); - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/package.html ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/package.html b/src/java/main/org/apache/zookeeper/server/package.html deleted file mode 100644 index 89a5d7e..0000000 --- a/src/java/main/org/apache/zookeeper/server/package.html +++ /dev/null @@ -1,101 +0,0 @@ - - - - -

ZooKeeper server theory of operation

-ZooKeeperServer is designed to work in standalone mode and also -be extensible so that it can be used to implement the quorum based -version of ZooKeeper. -

-ZooKeeper maintains a order when processing requests: -

    -
  • All requests will be processed in order. -
  • All responses will return in order. -
  • All watches will be sent in the order that the update takes place. -
-

-We will explain the three aspects of ZooKeeperServer: request processing, data -structure maintenance, and session tracking. - -

Request processing

- -Requests are received by the ServerCnxn. Demarshalling of a request is -done by ClientRequestHandler. After a request has been demarshalled, -ClientRequestHandler invokes the relevant method in ZooKeeper and marshals -the result. -

-If the request is just a query, it will be processed by ZooKeeper and returned. -Otherwise, the request will be validated and a transaction will be generated -and logged. This the request will then wait until the request has been logged -before continuing processing. -

-Requests are logged as a group. Transactions are queued up and the SyncThread -will process them at predefined intervals. (Currently 20ms) The SyncThread -interacts with ZooKeeperServer the txnQueue. Transactions are added to the -txnQueue of SyncThread via queueItem. When the transaction has been synced to -disk, its callback will be invoked which will cause the request processing to -be completed. - -

Data structure maintenance

- -ZooKeeper data is stored in-memory. Each znode is stored in a DataNode object. -This object is accessed through a hash table that maps paths to DataNodes. -DataNodes also organize themselves into a tree. This tree is only used for -serializing nodes. -

-We guarantee that changes to nodes are stored to non-volatile media before -responding to a client. We do this quickly by writing changes as a sequence -of transactions in a log file. Even though we flush transactions as a group, -we need to avoid seeks as much as possible. Also, since the server can fail -at any point, we need to be careful of partial records. -

-We address the above problems by -

    -
  • Pre-allocating 1M chunks of file space. This allows us to append to the -file without causing seeks to update file size. It also means that we need -to check for the end of the log by looking for a zero length transaction -rather than simply end of file. -
  • Writing a signature at the end of each transaction. When processing -transactions, we only use transactions that have a valid signature at the end. -
-

-As the server runs, the log file will grow quite large. To avoid long startup -times we periodically take a snapshot of the tree of DataNodes. We cannot -take the snapshot synchronously as the data takes a while to write out, so -instead we asynchronously write out the tree. This means that we end up -with a "corrupt" snapshot of the data tree. More formally if we define T -to be the real snapshot of the tree at the time we begin taking the snapshot -and l as the sequence of transactions that are applied to the tree between -the time the snapshot begins and the time the snapshot completes, we write -to disk T+l' where l' is a subset of the transactions in l. While we do not -have a way of figuring out which transactions make up l', it doesn't really -matter. T+l'+l = T+l since the transactions we log are idempotent (applying -the transaction multiple times has the same result as applying the transaction -once). So when we restore the snapshot we also play all transactions in the log -that occur after the snapshot was begun. We can easily figure out where to -start the replay because we start a new logfile when we start a snapshot. Both -the snapshot file and log file have a numeric suffix that represent the -transaction id that created the respective files. - -

Session tracking

-Rather than tracking sessions exactly, we track them in batches. That are -processed at fixed intervals. This is easier to implement than exact -session tracking and it is more efficient in terms of performance. It also -provides a small grace period for session renewal. - - \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java b/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java deleted file mode 100644 index 49fde24..0000000 --- a/src/java/main/org/apache/zookeeper/server/persistence/FilePadding.java +++ /dev/null @@ -1,112 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.persistence; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; - -public class FilePadding { - private static final Logger LOG; - private static long preAllocSize = 65536 * 1024; - private static final ByteBuffer fill = ByteBuffer.allocateDirect(1); - - static { - LOG = LoggerFactory.getLogger(FileTxnLog.class); - - String size = System.getProperty("zookeeper.preAllocSize"); - if (size != null) { - try { - preAllocSize = Long.parseLong(size) * 1024; - } catch (NumberFormatException e) { - LOG.warn(size + " is not a valid value for preAllocSize"); - } - } - } - - private long currentSize; - - /** - * Getter of preAllocSize has been added for testing - */ - public static long getPreAllocSize() { - return preAllocSize; - } - - /** - * method to allow setting preallocate size - * of log file to pad the file. - * - * @param size the size to set to in bytes - */ - public static void setPreallocSize(long size) { - preAllocSize = size; - } - - public void setCurrentSize(long currentSize) { - this.currentSize = currentSize; - } - - /** - * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position - * - * @param fileChannel the fileChannel of the file to be padded - * @throws IOException - */ - long padFile(FileChannel fileChannel) throws IOException { - long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); - if (currentSize != newFileSize) { - fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); - currentSize = newFileSize; - } - return currentSize; - } - - /** - * Calculates a new file size with padding. We only return a new size if - * the current file position is sufficiently close (less than 4K) to end of - * file and preAllocSize is > 0. - * - * @param position the point in the file we have written to - * @param fileSize application keeps track of the current file size - * @param preAllocSize how many bytes to pad - * @return the new file size. It can be the same as fileSize if no - * padding was done. - * @throws IOException - */ - // VisibleForTesting - public static long calculateFileSizeWithPadding(long position, long fileSize, long preAllocSize) { - // If preAllocSize is positive and we are within 4KB of the known end of the file calculate a new file size - if (preAllocSize > 0 && position + 4096 >= fileSize) { - // If we have written more than we have previously preallocated we need to make sure the new - // file size is larger than what we already have - if (position > fileSize) { - fileSize = position + preAllocSize; - fileSize -= fileSize % preAllocSize; - } else { - fileSize += preAllocSize; - } - } - - return fileSize; - } -} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/c0aa3b3f/src/java/main/org/apache/zookeeper/server/persistence/FileSnap.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileSnap.java b/src/java/main/org/apache/zookeeper/server/persistence/FileSnap.java deleted file mode 100644 index 1687277..0000000 --- a/src/java/main/org/apache/zookeeper/server/persistence/FileSnap.java +++ /dev/null @@ -1,255 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.server.persistence; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.zip.Adler32; -import java.util.zip.CheckedInputStream; -import java.util.zip.CheckedOutputStream; - -import org.apache.jute.BinaryInputArchive; -import org.apache.jute.BinaryOutputArchive; -import org.apache.jute.InputArchive; -import org.apache.jute.OutputArchive; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.zookeeper.server.DataTree; -import org.apache.zookeeper.server.util.SerializeUtils; - -/** - * This class implements the snapshot interface. - * it is responsible for storing, serializing - * and deserializing the right snapshot. - * and provides access to the snapshots. - */ -public class FileSnap implements SnapShot { - File snapDir; - private volatile boolean close = false; - private static final int VERSION=2; - private static final long dbId=-1; - private static final Logger LOG = LoggerFactory.getLogger(FileSnap.class); - public final static int SNAP_MAGIC - = ByteBuffer.wrap("ZKSN".getBytes()).getInt(); - - public static final String SNAPSHOT_FILE_PREFIX = "snapshot"; - - public FileSnap(File snapDir) { - this.snapDir = snapDir; - } - - /** - * deserialize a data tree from the most recent snapshot - * @return the zxid of the snapshot - */ - public long deserialize(DataTree dt, Map sessions) - throws IOException { - // we run through 100 snapshots (not all of them) - // if we cannot get it running within 100 snapshots - // we should give up - List snapList = findNValidSnapshots(100); - if (snapList.size() == 0) { - return -1L; - } - File snap = null; - boolean foundValid = false; - for (int i = 0; i < snapList.size(); i++) { - snap = snapList.get(i); - InputStream snapIS = null; - CheckedInputStream crcIn = null; - try { - LOG.info("Reading snapshot " + snap); - snapIS = new BufferedInputStream(new FileInputStream(snap)); - crcIn = new CheckedInputStream(snapIS, new Adler32()); - InputArchive ia = BinaryInputArchive.getArchive(crcIn); - deserialize(dt,sessions, ia); - long checkSum = crcIn.getChecksum().getValue(); - long val = ia.readLong("val"); - if (val != checkSum) { - throw new IOException("CRC corruption in snapshot : " + snap); - } - foundValid = true; - break; - } catch(IOException e) { - LOG.warn("problem reading snap file " + snap, e); - } finally { - if (snapIS != null) - snapIS.close(); - if (crcIn != null) - crcIn.close(); - } - } - if (!foundValid) { - throw new IOException("Not able to find valid snapshots in " + snapDir); - } - dt.lastProcessedZxid = Util.getZxidFromName(snap.getName(), SNAPSHOT_FILE_PREFIX); - return dt.lastProcessedZxid; - } - - /** - * deserialize the datatree from an inputarchive - * @param dt the datatree to be serialized into - * @param sessions the sessions to be filled up - * @param ia the input archive to restore from - * @throws IOException - */ - public void deserialize(DataTree dt, Map sessions, - InputArchive ia) throws IOException { - FileHeader header = new FileHeader(); - header.deserialize(ia, "fileheader"); - if (header.getMagic() != SNAP_MAGIC) { - throw new IOException("mismatching magic headers " - + header.getMagic() + - " != " + FileSnap.SNAP_MAGIC); - } - SerializeUtils.deserializeSnapshot(dt,ia,sessions); - } - - /** - * find the most recent snapshot in the database. - * @return the file containing the most recent snapshot - */ - public File findMostRecentSnapshot() throws IOException { - List files = findNValidSnapshots(1); - if (files.size() == 0) { - return null; - } - return files.get(0); - } - - /** - * find the last (maybe) valid n snapshots. this does some - * minor checks on the validity of the snapshots. It just - * checks for / at the end of the snapshot. This does - * not mean that the snapshot is truly valid but is - * valid with a high probability. also, the most recent - * will be first on the list. - * @param n the number of most recent snapshots - * @return the last n snapshots (the number might be - * less than n in case enough snapshots are not available). - * @throws IOException - */ - private List findNValidSnapshots(int n) throws IOException { - List files = Util.sortDataDir(snapDir.listFiles(), SNAPSHOT_FILE_PREFIX, false); - int count = 0; - List list = new ArrayList(); - for (File f : files) { - // we should catch the exceptions - // from the valid snapshot and continue - // until we find a valid one - try { - if (Util.isValidSnapshot(f)) { - list.add(f); - count++; - if (count == n) { - break; - } - } - } catch (IOException e) { - LOG.info("invalid snapshot " + f, e); - } - } - return list; - } - - /** - * find the last n snapshots. this does not have - * any checks if the snapshot might be valid or not - * @param the number of most recent snapshots - * @return the last n snapshots - * @throws IOException - */ - public List findNRecentSnapshots(int n) throws IOException { - List files = Util.sortDataDir(snapDir.listFiles(), SNAPSHOT_FILE_PREFIX, false); - int count = 0; - List list = new ArrayList(); - for (File f: files) { - if (count == n) - break; - if (Util.getZxidFromName(f.getName(), SNAPSHOT_FILE_PREFIX) != -1) { - count++; - list.add(f); - } - } - return list; - } - - /** - * serialize the datatree and sessions - * @param dt the datatree to be serialized - * @param sessions the sessions to be serialized - * @param oa the output archive to serialize into - * @param header the header of this snapshot - * @throws IOException - */ - protected void serialize(DataTree dt,Map sessions, - OutputArchive oa, FileHeader header) throws IOException { - // this is really a programmatic error and not something that can - // happen at runtime - if(header==null) - throw new IllegalStateException( - "Snapshot's not open for writing: uninitialized header"); - header.serialize(oa, "fileheader"); - SerializeUtils.serializeSnapshot(dt,oa,sessions); - } - - /** - * serialize the datatree and session into the file snapshot - * @param dt the datatree to be serialized - * @param sessions the sessions to be serialized - * @param snapShot the file to store snapshot into - */ - public synchronized void serialize(DataTree dt, Map sessions, File snapShot) - throws IOException { - if (!close) { - OutputStream sessOS = new BufferedOutputStream(new FileOutputStream(snapShot)); - CheckedOutputStream crcOut = new CheckedOutputStream(sessOS, new Adler32()); - //CheckedOutputStream cout = new CheckedOutputStream() - OutputArchive oa = BinaryOutputArchive.getArchive(crcOut); - FileHeader header = new FileHeader(SNAP_MAGIC, VERSION, dbId); - serialize(dt,sessions,oa, header); - long val = crcOut.getChecksum().getValue(); - oa.writeLong(val, "val"); - oa.writeString("/", "path"); - sessOS.flush(); - crcOut.close(); - sessOS.close(); - } - } - - /** - * synchronized close just so that if serialize is in place - * the close operation will block and will wait till serialize - * is done and will set the close flag - */ - @Override - public synchronized void close() throws IOException { - close = true; - } - - }