climate-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From good...@apache.org
Subject climate git commit: CLIMATE-874 - Remove Easy-OCW
Date Thu, 20 Oct 2016 22:16:27 GMT
Repository: climate
Updated Branches:
  refs/heads/master f9a9517cb -> fc520781d


CLIMATE-874 - Remove Easy-OCW


Project: http://git-wip-us.apache.org/repos/asf/climate/repo
Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fc520781
Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fc520781
Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fc520781

Branch: refs/heads/master
Commit: fc520781d218ad2ab6bade79cd3779d1fd6e0dee
Parents: f9a9517
Author: Alex Goodman <agoodm@users.noreply.github.com>
Authored: Wed Oct 19 20:49:18 2016 -0700
Committer: Alex Goodman <agoodm@users.noreply.github.com>
Committed: Thu Oct 20 13:58:33 2016 -0700

----------------------------------------------------------------------
 .travis.yml                         |  36 ++++-
 deps_py2.txt                        |  14 ++
 deps_py3.txt                        |  11 ++
 easy-ocw/install-osx.sh             | 242 -------------------------------
 easy-ocw/install-ubuntu.sh          | 157 --------------------
 easy-ocw/ocw-conda-dependencies.txt |  23 ---
 easy-ocw/ocw-pip-dependencies.txt   |  12 --
 ocw/dataset_loader.py               |  16 +-
 test.sh                             |   6 +-
 test_smoke.py                       |  14 +-
 10 files changed, 80 insertions(+), 451 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 10d9d17..1a50d8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,23 +17,49 @@
 language: python
 python:
   - "2.7"
-  - "3.2"
-  - "3.3"
   - "3.4"
   - "3.5"
 notifications:
   email: false
 install:
   - sudo apt-get update
-  - source easy-ocw/install-ubuntu.sh
-  - pip install python-coveralls
+  # We do this conditionally because it saves us some downloading if the
+  # version is the same.
+  - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
+      wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+      DEPS=deps_py2.txt;
+    else
+      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+      DEPS=deps_py3.txt;
+    fi
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  
+  # Useful for debugging any issues with conda
+  - conda info -a
+  
+  # Add conda-forge to search so all dependencies can be resolved
+  - conda config --add channels conda-forge
+  - conda create -q -n test-environment nose python=$TRAVIS_PYTHON_VERSION --file $DEPS
+
+  # Need this to exclude pydap test for python 3
+  - pip install nose-exclude
+  
+  # Install ocw
+  - source activate test-environment
+  - python setup.py install
+  
 # Workaround to get travis build working with matplotlib
 before_script:
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
   - sleep 3 # give xvfb some time to start
+
 script:
   - chmod a+x test.sh
-  - ./test.sh
+  - ./test.sh $TRAVIS_PYTHON_VERSION
 after_success:
   - coveralls

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py2.txt
----------------------------------------------------------------------
diff --git a/deps_py2.txt b/deps_py2.txt
new file mode 100644
index 0000000..c5cd8b6
--- /dev/null
+++ b/deps_py2.txt
@@ -0,0 +1,14 @@
+numpy
+scipy
+matplotlib
+basemap
+netcdf4
+h5py
+bottle
+pydap
+python-dateutil
+mock
+webtest
+myproxyclient
+esgf-pyclient
+podaacpy

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py3.txt
----------------------------------------------------------------------
diff --git a/deps_py3.txt b/deps_py3.txt
new file mode 100644
index 0000000..b5fbfa8
--- /dev/null
+++ b/deps_py3.txt
@@ -0,0 +1,11 @@
+numpy
+scipy
+matplotlib
+basemap
+netcdf4
+h5py
+bottle
+python-dateutil
+mock
+webtest
+podaacpy

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-osx.sh
----------------------------------------------------------------------
diff --git a/easy-ocw/install-osx.sh b/easy-ocw/install-osx.sh
deleted file mode 100755
index aeaeae6..0000000
--- a/easy-ocw/install-osx.sh
+++ /dev/null
@@ -1,242 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-export ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future
-
-help()
-{
-cat << ENDHELP
-
-Easy OCW assists with the building of the Apache Open Climate Workbench and its dependencies
-
-Flags:
-    -h  Display this help message.
-    -e  Install and configure a virtualenv environment before installation.
-    -q  Quiet install. User prompts are removed (when possible).
-
-It is recommended that you pass -e when running this script. If you don't, parts
-of this installation will pollute your global Python install. If you're unsure,
-pass -e just to be safe!
-
-ENDHELP
-}
-
-header()
-{
-    echo
-    echo $1
-}
-
-task()
-{
-    echo " - " $1
-}
-
-subtask()
-{
-    echo "     " $1
-}
-
-echo
-echo "---------------------------------------------------------------------------"
-echo "                         Welcome to Easy OCW"
-echo "---------------------------------------------------------------------------"
-echo
-
-WITH_VIRTUAL_ENV=0
-WITH_HOMEBREW=0
-WITH_INTERACT=1
-INIT_PWD=$PWD
-
-while getopts ":h :e :q" FLAG
-do
-    case $FLAG in
-        h)
-            help
-            exit 1
-            ;;
-        e)
-            WITH_VIRTUAL_ENV=1
-            ;;
-        q)
-            WITH_INTERACT=0
-            ;;
-        ?)
-            help
-            exit 1
-            ;;
-    esac
-done
-
-if [ $WITH_INTERACT == 1 ]; then
-cat << ENDINTRO
-A number of dependencies for OCW will now be installed. Please check the wiki
-for a complete list of dependencies. Additionally, please read the wiki for
-useful installation guidelines and information that may be pertinent to your
-situation. All of this can be found at http://s.apache.org/3p2
-
-ENDINTRO
-
-if [ $WITH_VIRTUAL_ENV != 1 ]; then
-cat << VIRTUALENV_WARNING
-$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput
sgr 0)
-It is highly recommended that you allow Easy OCW to install the dependencies
-into a virtualenv environment to ensure that your global Python install is
-not affected. If you're UNSURE, you should pass the -e flag
-to this script. If you aren't concerned, or you want to create your own
-virtualenv environment, then feel free to ignore this message.
-
-VIRTUALENV_WARNING
-
-read -p "Press [Yy] to begin installation with the flag -e $(tput setaf 2)[RECOMMENDED]$(tput
sgr 0)
-[OR] 
-Press [Nn] to continue with the normal installation..." yn
-case $yn in 
-    [Yy]* ) 
-            WITH_VIRTUAL_ENV=1
-            ;;
-    [Nn]* ) 
-            WITH_VIRTUAL_ENV=0 
-            ;;
-    * ) echo "Please answer yes or no.." ;;
-esac
-fi
-fi
-
-header "Checking for pip ..."
-command -v pip >/dev/null 2>&1 || { 
-    task "Unable to locate pip."
-    task "Installing Distribute"
-    curl http://python-distribute.org/distribute_setup.py | python >> install_log
-    subtask "done"
-
-    task "Installing Pip"
-    curl -O http://pypi.python.org/packages/source/p/pip/pip-1.2.1.tar.gz >> install_log
-    tar xzf pip-1.2.1.tar.gz >> install_log
-    cd pip-1.2.1/
-    python setup.py install >> install_log
-    subtask "done"
-}
-
-if [ $WITH_VIRTUAL_ENV == 1 ]; then
-    header "Setting up a virtualenv ..."
-
-    # Check if virtualenv is installed. If it's not, we'll install it for the user.
-    command -v virtualenv >/dev/null 2>&1 || { 
-        task "Installing virtualenv ..."
-        pip install virtualenv >> install_log
-        subtask "done"
-    }
-
-    #Read the path for installation from the user.
-    read -e -p "Enter the path where you want to install ocw..." ocw_path
-    #check if the file path exists.
-    while [ ! -d "$ocw_path" ];
-    do 
-        echo "Path not found..."
-        read -e -p "Please enter a valid path..." ocw_path
-    done
-
-    header "Checking for previously installed  ocw virtual environment..."
-    if [ -e $ocw_path/ocw/bin/python ]; then
-        echo "We found an existing 'ocw' virtualenv on your system in $ocw_path/ocw."
-        read -n1 -p "Do you want to replace it with a clean install? y/n :" replace 
-        if [ "$replace" == "y" ]; then
-            echo ""
-            echo "$(tput setaf 1)[WARNING!] this will delete all file and data in $ocw_path/ocw
on your system.$(tput sgr 0)"
-            read -n1 -p "To confirm and proceed type y or n to quit:" confirm
-            if [ "$confirm" == "y" ]; then
-                echo ""
-                echo "Deleting contents of $ocw_path/ocw" >> install_log
-                rm -rf $ocw_path/ocw
-            else
-                echo ""
-                echo "Stopping Open Climate Workbench Installation"
-                exit
-            fi
-        else
-            echo ""
-            echo "Stopping Open Climate Workbench Installation because an existing 'ocw'"
-            echo "virtual environment already exists."
-            exit
-        fi
-    fi
-
-    cd $ocw_path
-    # Create a new environment for OCW work
-    task "Creating a new environment in ~/ocw..."
-    virtualenv ocw >> install_log
-    source $ocw_path/ocw/bin/activate >> install_log
-    cd $INIT_PWD
-    subtask "done"
-fi
-
-header "Installing conda for package management ..."
-
-pip install conda >> install_log
-conda init >> install_log
-
-
-header "Installing dependencies with conda ..."
-echo | conda install --file ocw-conda-dependencies.txt
-
-# We only use conda for the annoying dependencies like numpy,
-# scipy, matplotlib, and basemap. For everything else, we stick
-# with pip.
-header "Installing additional Python packages"
-pip install -r ocw-pip-dependencies.txt >> install_log
-
-if [ $WITH_VIRTUAL_ENV == 1 ]; then
-    echo "***POST INSTALLATION NOTE***
-
-If you are familiar with virtualenv you should know that activating
-the new 'ocw' environment is different because we use conda to install
-packages.  An example of the command you want to run is listed in the
-alias below.
-
-To make it easier to change into the 'ocw' virtualenv add the
-following alias to your ~/.bash_profile
-
-    alias ocw='source $ocw_path/ocw/bin/activate $ocw_path/ocw/'
-
-When you want to use ocw in the future, you just have to type 'ocw'
-in your terminal."
-else
-    echo "***POST INSTALLATION NOTE***
-
-If you have run this script within your own virtualenv you need to know
-a couple of caveats/side effects that are caused by using conda to install
-packages within the virtualenv.
-
-- Virtualenv wrapper will throw errors like those outlined here:
-https://issues.apache.org/jira/browse/CLIMATE-451
-
-- You will not be able to 'activate' the environment using the normal
-virtualenv command, you must instead use the conda activate command as follows:
-
-source path/to/your_env/bin/activate path/to/your_env
-
-Example:  (assuming your env is in ~/.virtualenv/ocw)
-
-source ~/.virtualenv/ocw/bin/activate ~/.virtualenv/ocw
-
-"
-
-fi
-

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-ubuntu.sh
----------------------------------------------------------------------
diff --git a/easy-ocw/install-ubuntu.sh b/easy-ocw/install-ubuntu.sh
deleted file mode 100755
index c9db70c..0000000
--- a/easy-ocw/install-ubuntu.sh
+++ /dev/null
@@ -1,157 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-help()
-{
-cat << ENDHELP
-
-Easy OCW assists with the building of the Apache Open Climate Workbench and 
-its dependencies.
-
-Flags:
-    -h  Display this help message.
-
-N.B. This install script has been tested against Ubuntu 12.04 and 14.04.
-Please report problems with this script to dev@climate.apache.org
-ENDHELP
-}
-
-header()
-{
-    echo
-    echo $1
-}
-
-task()
-{
-    echo " - " $1
-}
-
-subtask()
-{
-    echo "     " $1
-}
-
-echo
-echo "---------------------------------------------------------------------------"
-echo "                         Welcome to Easy OCW"
-echo "---------------------------------------------------------------------------"
-echo
-
-# Find absolute path to the easy-ocw directory
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-cd $DIR
-ocw_path="$DIR/../climate"
-
-while getopts ":h" FLAG
-do
-    case $FLAG in
-        h)
-            help
-            exit 1
-            ;;
-        ?)
-            help
-            exit 1
-            ;;
-    esac
-done
-
-cat << ENDINTRO
-A number of dependencies for OCW will now be installed. Please check the wiki
-for a complete list of dependencies. Additionally, please read the wiki for
-useful installation guidelines and information that may be pertinent to your
-situation. All of this can be found at http://s.apache.org/3p2
-ENDINTRO
-
-cat << VIRTUALENV_WARNING
-$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput
sgr 0)
-Easy OCW will automatically create a conda environment "venv-ocw".
-All the dependencies will be installed in the virtual environment
-to ensure that your global Python install is not affected.
-
-Please activate the virtual environment venv-ocw after the script has completed to use OCW.
-
-To activate the virtual environment execute
-> source active venv-ocw
-
-To deactivate the virtual environment execute
-> source deactivate
-
-VIRTUALENV_WARNING
-
-echo "Easy-OCW script logs" > install_log
-
-# Install Continuum Analytics Miniconda Python distribution. This gives
-# almost all the dependencies that OCW needs in a single, easy to
-# install package.
-header "Installing Miniconda Python distribution ..."
-
-MACHINE_TYPE=`uname -m`
-if [ ${MACHINE_TYPE} == 'x86_64' ]; then
-	link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"
-else
-	link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh"
-fi
-
-header "Checking for conda ..."
-command -v conda >/dev/null 2>&1 || { 
-    task "Unable to locate conda."
-    if ! [ -f Miniconda-latest-linux.sh ]; then
-		task "Downloading Miniconda ..."
-		wget -O Miniconda-latest-linux.sh $link >> install_log 2>&1
-		subtask "done"
-    fi
-	task "Installing ..."
-	bash Miniconda-latest-linux.sh -b -p $HOME/miniconda
-	export PATH="$HOME/miniconda/bin/:$PATH"
-	subtask "done"
-}
-
-header "Creating venv-ocw environment and installing dependencies"
-conda create --name venv-ocw -y --file ocw-conda-dependencies.txt
-task "Activating venv-ocw virtual environment"
-source activate venv-ocw
-
-conda config --set always_yes yes
-conda update -q conda
-
-header "Conda Information"
-echo "------------------------------------------------------------------"
-conda info -a
-
-echo 
-task "Installing python-dev"
-sudo apt-get -y install python-dev >> install_log 2>&1
-subtask "done"
-
-cd $DIR
-
-header "Installing remaining dependencies via pip"
-pip install -r ocw-pip-dependencies.txt
-subtask "done"
-
-header "Installing ocw module"
-
-cd ..
-python setup.py install
-subtask "Finished installing OCW module"
-
-header "For any issues with installation please contact dev@climate.apache.org"

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-conda-dependencies.txt
----------------------------------------------------------------------
diff --git a/easy-ocw/ocw-conda-dependencies.txt b/easy-ocw/ocw-conda-dependencies.txt
deleted file mode 100644
index 665be4d..0000000
--- a/easy-ocw/ocw-conda-dependencies.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-numpy>=1.10.4
-scipy>=0.17.0
-matplotlib>=1.5.1
-basemap>=1.0.7
-netcdf4>=1.2.2
-h5py>=2.6.0

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-pip-dependencies.txt
----------------------------------------------------------------------
diff --git a/easy-ocw/ocw-pip-dependencies.txt b/easy-ocw/ocw-pip-dependencies.txt
deleted file mode 100644
index 867c801..0000000
--- a/easy-ocw/ocw-pip-dependencies.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-requests>=2.10.0
-bottle>=0.12.9
-pydap>=3.1.1
-webtest>=2.0.21
-nose>=1.3.7
-sphinx>=1.4.4
-sphinxcontrib-httpdomain>=1.5.0
-esgf-pyclient>=0.1.6
-python-dateutil>=2.5.3
-mock>=2.0.0
-myproxyclient>=1.4.3
-podaacpy>=1.0.2

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/ocw/dataset_loader.py
----------------------------------------------------------------------
diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py
index ed8fdb0..fdcbb1b 100644
--- a/ocw/dataset_loader.py
+++ b/ocw/dataset_loader.py
@@ -21,11 +21,9 @@ Classes:
 '''
 
 import ocw.data_source.local as local
-import ocw.data_source.esgf as esgf
 import ocw.data_source.rcmed as rcmed
-import ocw.data_source.dap as dap
 import ocw.data_source.podaac_datasource as podaac
-
+import warnings
 
 class DatasetLoader:
     '''Generate a list of OCW Dataset objects from a variety of sources.'''
@@ -95,11 +93,19 @@ class DatasetLoader:
         self._source_loaders = {
             'local': local.load_multiple_files,
             'local_split': local.load_dataset_from_multiple_netcdf_files,
-            'esgf': esgf.load_dataset,
             'rcmed': rcmed.parameter_dataset,
-            'dap': dap.load,
             'podaac': podaac.load_dataset
         }
+        
+        # Exclude esgf and dap for python 3 until they are compatible
+        try:
+            import ocw.data_source.esgf as esgf
+            import ocw.data_source.dap as dap
+            self._source_loaders['dap'] = dap.load
+            self._source_loaders['esgf'] = esgf.load_dataset
+        except ImportError:
+            warnings.warn('dap and esgf loaders missing. If these are needed, '
+                          'fallback to python 2.7.x.')
 
     def add_source_loader(self, loader_name, loader_func):
         '''

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test.sh
----------------------------------------------------------------------
diff --git a/test.sh b/test.sh
index 0ea488d..cc73d04 100755
--- a/test.sh
+++ b/test.sh
@@ -25,7 +25,11 @@ echo ""
 # nosetests config file should be in home directory
 cp .noserc $HOME/.noserc
 
+# Exclude dap tests if using python 3
+if [[ "$1" != "2.7" ]]; then
+  export NOSE_EXCLUDE_TESTS=ocw.tests.test_dap.TestDap
+fi
+
 echo "---------------- Running Unit Tests ---------------"
 nosetests
 echo "---------------- All Tests successfully completed ---------------"
-

http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test_smoke.py
----------------------------------------------------------------------
diff --git a/test_smoke.py b/test_smoke.py
index b1599df..f08d72a 100644
--- a/test_smoke.py
+++ b/test_smoke.py
@@ -21,9 +21,13 @@ from ocw.tests.test_local import create_netcdf_object
 from ocw.data_source import local
 from ocw import dataset_processor as dsp
 import os
+import six
+
+if six.PY3:
+    DEPENDENCIES_FILE = 'deps_py3.txt'
+else:
+    DEPENDENCIES_FILE = 'deps_py2.txt'
 
-PIP_DEPENDENCIES_FILE = 'easy-ocw/ocw-pip-dependencies.txt'
-CONDA_DEPENDENCIES_FILE = 'easy-ocw/ocw-conda-dependencies.txt'
 SUCCESS_MARK = '\033[92m' + u'\u2713' + '\033[0m'
 FAILURE_MARK = '\033[91m' + u'\u274C' + '\033[0m'
 
@@ -123,11 +127,9 @@ def end(dataset=None):
 
 
 def main():
-    pip_file = open(PIP_DEPENDENCIES_FILE, 'r')
-    conda_file = open(CONDA_DEPENDENCIES_FILE, 'r')
+    dep_file = open(DEPENDENCIES_FILE, 'r')
     print("Checking installed dependencies\n")
-    check_dependencies(conda_file)
-    check_dependencies(pip_file)
+    check_dependencies(dep_file)
     success("\nDependencies")
     dataset = check_dataset_loading()
     check_some_dataset_functions(dataset)


Mime
View raw message