Return-Path: Delivered-To: apmail-lucene-solr-commits-archive@minotaur.apache.org Received: (qmail 14982 invoked from network); 4 Feb 2010 01:08:33 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Feb 2010 01:08:33 -0000 Received: (qmail 19573 invoked by uid 500); 4 Feb 2010 01:08:33 -0000 Delivered-To: apmail-lucene-solr-commits-archive@lucene.apache.org Received: (qmail 19486 invoked by uid 500); 4 Feb 2010 01:08:33 -0000 Mailing-List: contact solr-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: solr-dev@lucene.apache.org Delivered-To: mailing list solr-commits@lucene.apache.org Received: (qmail 19477 invoked by uid 99); 4 Feb 2010 01:08:33 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Feb 2010 01:08:33 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Feb 2010 01:08:26 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id EC5BD23888EC; Thu, 4 Feb 2010 01:08:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r906316 - in /lucene/solr/branches/cloud/src: java/org/apache/solr/cloud/ java/org/apache/solr/core/ java/org/apache/solr/handler/component/ test/org/apache/solr/cloud/ webapp/web/admin/ Date: Thu, 04 Feb 2010 01:08:05 -0000 To: solr-commits@lucene.apache.org From: markrmiller@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20100204010805.EC5BD23888EC@eris.apache.org> Author: markrmiller Date: Thu Feb 4 01:08:04 2010 New Revision: 906316 URL: http://svn.apache.org/viewvc?rev=906316&view=rev Log: remove non common dependencies from zkController for possible sharing with solrj Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java (original) +++ lucene/solr/branches/cloud/src/java/org/apache/solr/cloud/ZkController.java Thu Feb 4 01:08:04 2010 @@ -17,10 +17,8 @@ * limitations under the License. */ -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.InetAddress; import java.util.HashMap; import java.util.HashSet; @@ -34,14 +32,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import javax.xml.parsers.ParserConfigurationException; - import org.apache.solr.cloud.SolrZkClient.OnReconnect; import org.apache.solr.common.SolrException; -import org.apache.solr.core.SolrConfig; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrResourceLoader; -import org.apache.solr.schema.IndexSchema; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; @@ -50,7 +42,6 @@ import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; /** * Handle ZooKeeper interactions. @@ -219,35 +210,6 @@ } /** - * Load SolrConfig from ZooKeeper. - * - * TODO: consider *many* cores firing up at once and loading the same files - * from ZooKeeper - * - * @param resourceLoader - * @param solrConfigFileName - * @return - * @throws IOException - * @throws ParserConfigurationException - * @throws SAXException - * @throws InterruptedException - * @throws KeeperException - */ - public SolrConfig getConfig(String zkConfigName, String solrConfigFileName, - SolrResourceLoader resourceLoader) throws IOException, - ParserConfigurationException, SAXException, KeeperException, - InterruptedException { - byte[] config = zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName + "/" - + solrConfigFileName, null, null); - InputStream is = new ByteArrayInputStream(config); - SolrConfig cfg = solrConfigFileName == null ? new SolrConfig( - resourceLoader, SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig( - resourceLoader, solrConfigFileName, is); - - return cfg; - } - - /** * @param zkConfigName * @param fileName * @return @@ -256,7 +218,7 @@ */ public byte[] getConfigFileData(String zkConfigName, String fileName) throws KeeperException, InterruptedException { - return zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName, null, null); + return zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName + "/" + fileName, null, null); } // TODO: consider how this is done @@ -281,29 +243,6 @@ return hostName; } - /** - * Load IndexSchema from ZooKeeper. - * - * TODO: consider *many* cores firing up at once and loading the same files - * from ZooKeeper - * - * @param resourceLoader - * @param schemaName - * @param config - * @return - * @throws InterruptedException - * @throws KeeperException - */ - public IndexSchema getSchema(String zkConfigName, String schemaName, - SolrConfig config, SolrResourceLoader resourceLoader) - throws KeeperException, InterruptedException { - byte[] configBytes = zkClient.getData(CONFIGS_ZKNODE + "/" + zkConfigName - + "/" + schemaName, null, null); - InputStream is = new ByteArrayInputStream(configBytes); - IndexSchema schema = new IndexSchema(config, schemaName, is); - return schema; - } - public SolrZkClient getZkClient() { return zkClient; } @@ -528,23 +467,20 @@ } /** - * Register shard. A SolrCore calls this on startup to register with - * ZooKeeper. + * Register shard with ZooKeeper. * - * @param core SolrCore to register as a shard + * @param coreName + * @param cloudDesc * @param forcePropsUpdate update solr.xml core props even if the shard is already registered - * * @throws IOException * @throws KeeperException * @throws InterruptedException */ - public void register(SolrCore core, boolean forcePropsUpdate) throws IOException, + public void register(String coreName, CloudDescriptor cloudDesc, boolean forcePropsUpdate) throws IOException, KeeperException, InterruptedException { - String coreName = core.getCoreDescriptor().getName(); String shardUrl = localHostName + ":" + localHostPort + "/" + localHostContext + "/" + coreName; - CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor(); String collection = cloudDesc.getCollectionName(); String shardsZkPath = COLLECTIONS_ZKNODE + "/" + collection + SHARDS_ZKNODE + "/" + cloudDesc.getShardId(); @@ -556,7 +492,7 @@ } if (log.isInfoEnabled()) { - log.info("Register shard - core:" + core.getName() + " address:" + log.info("Register shard - core:" + coreName + " address:" + shardUrl); } @@ -596,9 +532,10 @@ } /** - * @param core + * @param coreName + * @param cloudDesc */ - public void unregister(SolrCore core) { + public void unregister(String coreName, CloudDescriptor cloudDesc) { // TODO : perhaps mark the core down in zk? } Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java (original) +++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreContainer.java Thu Feb 4 01:08:04 2010 @@ -79,7 +79,7 @@ protected String solrDataDirOverride; protected String zkPortOverride; private String testShardIdOverride; - private ZkController zooKeeperController; + private ZkController zkController; private SolrZkServer zkServer; private String zkHost; @@ -124,7 +124,7 @@ if (zkRun != null && zkServer.getServers().size() > 1) zkClientConnectTimeout = 24 * 60 * 60 * 1000; // 1 day for embedded ensemble - zooKeeperController = new ZkController(zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext); + zkController = new ZkController(zookeeperHost, zkClientTimeout, zkClientConnectTimeout, host, hostPort, hostContext); String confDir = System.getProperty("bootstrap_confdir"); if(confDir != null) { @@ -133,7 +133,7 @@ throw new IllegalArgumentException("bootstrap_confdir must be a directory of configuration files"); } String confName = System.getProperty("bootstrap_confname", "configuration1"); - zooKeeperController.uploadConfigDir(dir, confName); + zkController.uploadConfigDir(dir, confName); } } catch (InterruptedException e) { // Restore the interrupted status @@ -385,7 +385,7 @@ if (opt != null) { p.setSchemaName(opt); } - if (zooKeeperController != null) { + if (zkController != null) { opt = DOMUtil.getAttr(node, "shardId", null); if (testShardIdOverride != null && name.equals("")) { @@ -428,11 +428,11 @@ } - if(zooKeeperController != null) { + if(zkController != null) { try { - synchronized (zooKeeperController) { - zooKeeperController.addShardZkNodeWatches(); - zooKeeperController.updateCloudState(true); + synchronized (zkController) { + zkController.addShardZkNodeWatches(); + zkController.updateCloudState(true); } } catch (InterruptedException e) { // Restore the interrupted status @@ -474,8 +474,8 @@ } cores.clear(); } finally { - if(zooKeeperController != null) { - zooKeeperController.close(); + if(zkController != null) { + zkController.close(); } if (zkServer != null) { zkServer.stop(); @@ -519,9 +519,9 @@ core.setName(name); } - if (zooKeeperController != null) { + if (zkController != null) { try { - zooKeeperController.register(core, true); + zkController.register(core.getName(), core.getCoreDescriptor().getCloudDescriptor(), true); } catch (InterruptedException e) { // Restore the interrupted status Thread.currentThread().interrupt(); @@ -583,21 +583,21 @@ SolrConfig config = null; String zkConfigName = null; - if(zooKeeperController == null) { + if(zkController == null) { solrLoader = new SolrResourceLoader(instanceDir, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties())); config = new SolrConfig(solrLoader, dcore.getConfigName(), null); } else { try { String collection = dcore.getCloudDescriptor().getCollectionName(); - zooKeeperController.createCollectionZkNode(collection); - zkConfigName = zooKeeperController.readConfigName(collection); + zkController.createCollectionZkNode(collection); + zkConfigName = zkController.readConfigName(collection); if (zkConfigName == null) { log.error("Could not find config name for collection:" + collection); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "Could not find config name for collection:" + collection); } - solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()), zooKeeperController); - config = zooKeeperController.getConfig(zkConfigName, dcore.getConfigName(), solrLoader); + solrLoader = new ZkSolrResourceLoader(instanceDir, zkConfigName, libLoader, getCoreProps(instanceDir, dcore.getPropertiesName(),dcore.getCoreProperties()), zkController); + config = getSolrConfigFromZk(zkConfigName, dcore.getConfigName(), solrLoader); } catch (KeeperException e) { log.error("", e); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, @@ -612,7 +612,7 @@ } IndexSchema schema = null; if (indexSchemaCache != null) { - if (zooKeeperController != null) { + if (zkController != null) { File schemaFile = new File(dcore.getSchemaName()); if (!schemaFile.isAbsolute()) { schemaFile = new File(solrLoader.getInstanceDir() + "conf" @@ -638,9 +638,9 @@ } } if(schema == null){ - if(zooKeeperController != null) { + if(zkController != null) { try { - schema = zooKeeperController.getSchema(zkConfigName, dcore.getSchemaName(), config, solrLoader); + schema = getSchemaFromZk(zkConfigName, dcore.getSchemaName(), config, solrLoader); } catch (KeeperException e) { log.error("", e); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, @@ -1006,10 +1006,41 @@ throw xforward; } } - + public String getSolrHome() { return solrHome; } + + public boolean isZooKeeperAware() { + return zkController != null; + } + + public ZkController getZkController() { + return zkController; + } + + private SolrConfig getSolrConfigFromZk(String zkConfigName, String solrConfigFileName, + SolrResourceLoader resourceLoader) throws IOException, + ParserConfigurationException, SAXException, KeeperException, + InterruptedException { + byte[] config = zkController.getConfigFileData(zkConfigName, solrConfigFileName); + InputStream is = new ByteArrayInputStream(config); + SolrConfig cfg = solrConfigFileName == null ? new SolrConfig( + resourceLoader, SolrConfig.DEFAULT_CONF_FILE, is) : new SolrConfig( + resourceLoader, solrConfigFileName, is); + + return cfg; + } + + private IndexSchema getSchemaFromZk(String zkConfigName, String schemaName, + SolrConfig config, SolrResourceLoader resourceLoader) + throws KeeperException, InterruptedException { + byte[] configBytes = zkController.getConfigFileData(zkConfigName, schemaName); + InputStream is = new ByteArrayInputStream(configBytes); + IndexSchema schema = new IndexSchema(config, schemaName, is); + return schema; + } + private static final String DEF_SOLR_XML ="\n" + "\n" + " \n" + @@ -1017,12 +1048,5 @@ " \n" + ""; - public boolean isZooKeeperAware() { - return zooKeeperController != null; - } - - public ZkController getZooKeeperController() { - return zooKeeperController; - } } Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java (original) +++ lucene/solr/branches/cloud/src/java/org/apache/solr/core/CoreDescriptor.java Thu Feb 4 01:08:04 2010 @@ -43,11 +43,11 @@ this.coreContainer = coreContainer; this.name = name; - if(coreContainer.getZooKeeperController() != null) { + if(coreContainer.getZkController() != null) { this.cloudDesc = new CloudDescriptor(); // cloud collection defaults to core name cloudDesc.setCollectionName(name == "" ? coreContainer.getDefaultCoreName() : name); - this.cloudDesc.setShardId(coreContainer.getZooKeeperController().getNodeName() + "_" + name); + this.cloudDesc.setShardId(coreContainer.getZkController().getNodeName() + "_" + name); } if (name == null) { Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java (original) +++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryComponent.java Thu Feb 4 01:08:04 2010 @@ -132,7 +132,7 @@ Map slices = null; CoreDescriptor coreDescriptor = req.getCore().getCoreDescriptor(); CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor(); - ZkController zkController = coreDescriptor.getCoreContainer().getZooKeeperController(); + ZkController zkController = coreDescriptor.getCoreContainer().getZkController(); if (shards != null) { Modified: lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original) +++ lucene/solr/branches/cloud/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Thu Feb 4 01:08:04 2010 @@ -173,7 +173,7 @@ boolean exists = false; //nocommit: double check this how we want to handle this // check if using ZooKeeper - ZkController zooKeeperController = core.getCoreDescriptor().getCoreContainer().getZooKeeperController(); + ZkController zooKeeperController = core.getCoreDescriptor().getCoreContainer().getZkController(); if(zooKeeperController != null) { // TODO : shouldn't have to keep reading the config name when it has been read before exists = zooKeeperController.configFileExists(zooKeeperController.readConfigName(core.getCoreDescriptor().getCloudDescriptor().getCollectionName()), f); Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java (original) +++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/BasicZkTest.java Thu Feb 4 01:08:04 2010 @@ -29,7 +29,7 @@ // test using ZooKeeper assertTrue("Not using ZooKeeper", h.getCoreContainer().isZooKeeperAware()); - ZkController zkController = h.getCoreContainer().getZooKeeperController(); + ZkController zkController = h.getCoreContainer().getZkController(); // test merge factor picked up SolrCore core = h.getCore(); Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java (original) +++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/CloudStateUpdateTest.java Thu Feb 4 01:08:04 2010 @@ -122,7 +122,7 @@ // slight pause - TODO: takes an oddly long amount of time to schedule tasks with almost no delay ... Thread.sleep(5000); - ZkController zkController2 = container2.getZooKeeperController(); + ZkController zkController2 = container2.getZkController(); String host = zkController2.getHostName(); Modified: lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java (original) +++ lucene/solr/branches/cloud/src/test/org/apache/solr/cloud/DistributedZkFailuresTest.java Thu Feb 4 01:08:04 2010 @@ -101,7 +101,7 @@ @Override public void doTest() throws Exception { - ZkController zkController = h.getCoreContainer().getZooKeeperController(); + ZkController zkController = h.getCoreContainer().getZkController(); del("*:*"); // nocommit : test too long Modified: lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp URL: http://svn.apache.org/viewvc/lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp?rev=906316&r1=906315&r2=906316&view=diff ============================================================================== --- lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp (original) +++ lucene/solr/branches/cloud/src/webapp/web/admin/zookeeper.jsp Thu Feb 4 01:08:04 2010 @@ -101,7 +101,7 @@ this.addr = addr; if (addr == null) { - ZkController controller = core.getCoreDescriptor().getCoreContainer().getZooKeeperController(); + ZkController controller = core.getCoreDescriptor().getCoreContainer().getZkController(); if (controller != null) { // this core is zk enabled keeperAddr = controller.getZkServerAddress();