tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject tajo git commit: TAJO-1323: Cleanup the unstable test case. (jinho)
Date Mon, 02 Feb 2015 05:56:59 GMT
Repository: tajo
Updated Branches:
  refs/heads/master e656ee287 -> fd73074f2


TAJO-1323: Cleanup the unstable test case. (jinho)

Closes #368


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd73074f
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd73074f
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd73074f

Branch: refs/heads/master
Commit: fd73074f2198f92516a3eb8f5e786d31c1c071a5
Parents: e656ee2
Author: jhkim <jhkim@apache.org>
Authored: Mon Feb 2 14:56:14 2015 +0900
Committer: jhkim <jhkim@apache.org>
Committed: Mon Feb 2 14:56:14 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  2 +
 .../cli/tsql/DefaultTajoCliOutputFormatter.java |  3 +-
 .../java/org/apache/tajo/querymaster/Stage.java | 23 ++++-
 .../org/apache/tajo/TajoTestingCluster.java     | 96 ++++++++------------
 .../org/apache/tajo/cli/tsql/TestTajoCli.java   | 42 +++------
 .../tajo/engine/query/TestInsertQuery.java      |  3 +-
 .../apache/tajo/querymaster/TestKillQuery.java  | 20 +++-
 .../TestTajoCli/testNonForwardQueryPause.result |  3 +-
 8 files changed, 98 insertions(+), 94 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 4a3715b..8466a38 100644
--- a/CHANGES
+++ b/CHANGES
@@ -327,6 +327,8 @@ Release 0.10.0 - unreleased
 
   TASKS
 
+    TAJO-1323: Cleanup the unstable test case. (jinho)
+
     TAJO-1295: Remove legacy worker.dataserver package and its unit tests.
     (hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
index 17c94b9..5cbe77b 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/tsql/DefaultTajoCliOutputFormatter.java
@@ -37,6 +37,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
   private boolean printPause;
   private boolean printErrorTrace;
   private String nullChar;
+  public static char QUIT_COMMAND = 'q';
 
   @Override
   public void init(TajoCli.TajoCliContext context) {
@@ -123,7 +124,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter
{
         }
         sout.flush();
         if (sin != null) {
-          if (sin.read() == 'q') {
+          if (sin.read() == QUIT_COMMAND) {
             endOfTuple = false;
             sout.println();
             break;

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
index 208d4a6..5673d5b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
+++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java
@@ -185,7 +185,7 @@ public class Stage implements EventHandler<StageEvent> {
 
           // Transitions from KILL_WAIT state
           .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
-              StageEventType.SQ_CONTAINER_ALLOCATED,
+              EnumSet.of(StageEventType.SQ_START, StageEventType.SQ_CONTAINER_ALLOCATED),
               CONTAINERS_CANCEL_TRANSITION)
           .addTransition(StageState.KILL_WAIT, StageState.KILL_WAIT,
               EnumSet.of(StageEventType.SQ_KILL), new KillTasksTransition())
@@ -414,6 +414,18 @@ public class Stage implements EventHandler<StageEvent> {
     return totalScheduledObjectsCount;
   }
 
+  public int getKilledObjectCount() {
+    return killedObjectCount;
+  }
+
+  public int getFailedObjectCount() {
+    return failedObjectCount;
+  }
+
+  public int getCompletedTaskCount() {
+    return completedTaskCount;
+  }
+
   public ExecutionBlock getBlock() {
     return block;
   }
@@ -793,7 +805,14 @@ public class Stage implements EventHandler<StageEvent> {
                                 stage.taskScheduler.start();
                                 allocateContainers(stage);
                               } else {
-                                stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_KILL));
+                                /* all tasks are killed before stage are inited */
+                                if (stage.getTotalScheduledObjectsCount() == stage.getCompletedTaskCount())
{
+                                  stage.eventHandler.handle(
+                                      new StageEvent(stage.getId(), StageEventType.SQ_STAGE_COMPLETED));
+                                } else {
+                                  stage.eventHandler.handle(
+                                      new StageEvent(stage.getId(), StageEventType.SQ_KILL));
+                                }
                               }
                             }
                           } catch (Throwable e) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
index 8714fc4..0d3d660 100644
--- a/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -77,17 +77,13 @@ public class TajoTestingCluster {
   private TajoMaster tajoMaster;
   private List<TajoWorker> tajoWorkers = new ArrayList<TajoWorker>();
   private boolean standbyWorkerMode = false;
+  private boolean isDFSRunning = false;
+  private boolean isTajoClusterRunning = false;
+  private boolean isCatalogServerRunning = false;
 
-	// If non-null, then already a cluster running.
 	private File clusterTestBuildDir = null;
 
 	/**
-	 * System property key to get test directory value.
-	 * Name is as it is because mini dfs has hard-codings to put test data here.
-	 */
-	public static final String TEST_DIRECTORY_KEY = MiniDFSCluster.PROP_TEST_BUILD_DATA;
-
-	/**
 	 * Default parent directory for test output.
 	 */
 	public static final String DEFAULT_TEST_DIRECTORY = "target/test-data";
@@ -111,6 +107,7 @@ public class TajoTestingCluster {
     this.conf = new TajoConf();
     this.conf.setBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE, masterHaEMode);
 
+    initTestDir();
     setTestingFlagProperties();
     initPropertiesAndConfigs();
   }
@@ -177,22 +174,19 @@ public class TajoTestingCluster {
 		return this.conf;
 	}
 
-	public void initTestDir() {
-		if (System.getProperty(TEST_DIRECTORY_KEY) == null) {
-			clusterTestBuildDir = setupClusterTestBuildDir();
-			System.setProperty(TEST_DIRECTORY_KEY,
-          clusterTestBuildDir.getAbsolutePath());
-		}
-	}
+  public void initTestDir() {
+    if (clusterTestBuildDir == null) {
+      clusterTestBuildDir = setupClusterTestBuildDir();
+    }
+  }
 
 	/**
 	 * @return Where to write test data on local filesystem; usually
 	 * {@link #DEFAULT_TEST_DIRECTORY}
 	 * @see #setupClusterTestBuildDir()
 	 */
-	public static File getTestDir() {
-		return new File(System.getProperty(TEST_DIRECTORY_KEY,
-			DEFAULT_TEST_DIRECTORY));
+	public File getTestDir() {
+		return clusterTestBuildDir;
 	}
 
 	/**
@@ -202,10 +196,10 @@ public class TajoTestingCluster {
 	 * @see #setupClusterTestBuildDir()
 	 */
 	public static File getTestDir(final String subdirName) {
-		return new File(getTestDir(), subdirName);
+		return new File(new File(DEFAULT_TEST_DIRECTORY), subdirName);
   }
 
-	public File setupClusterTestBuildDir() {
+	public static File setupClusterTestBuildDir() {
 		String randomStr = UUID.randomUUID().toString();
 		String dirStr = getTestDir(randomStr).toString();
 		File dir = new File(dirStr).getAbsoluteFile();
@@ -243,9 +237,6 @@ public class TajoTestingCluster {
                                             File dir,
                                             final String hosts[])
       throws IOException {
-    if (dir == null) {
-      dir = setupClusterTestBuildDir();
-    }
 
     conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dir.toString());
     conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
@@ -263,7 +254,7 @@ public class TajoTestingCluster {
     this.defaultFS = this.dfsCluster.getFileSystem();
     this.conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultFS.getUri().toString());
     this.conf.setVar(TajoConf.ConfVars.ROOT_DIR, defaultFS.getUri() + "/tajo");
-
+    isDFSRunning = true;
     return this.dfsCluster;
   }
 
@@ -300,22 +291,20 @@ public class TajoTestingCluster {
   // Catalog Section
   ////////////////////////////////////////////////////////
   public MiniCatalogServer startCatalogCluster() throws Exception {
-    TajoConf c = getConfiguration();
+    if(isCatalogServerRunning) throw new IOException("Catalog Cluster already running");
 
-    if(clusterTestBuildDir == null) {
-      clusterTestBuildDir = setupClusterTestBuildDir();
-    }
+    TajoConf c = getConfiguration();
 
     conf.set(CatalogConstants.STORE_CLASS, "org.apache.tajo.catalog.store.MemStore");
     conf.set(CatalogConstants.CATALOG_URI, "jdbc:derby:" + clusterTestBuildDir.getAbsolutePath()
+ "/db");
-    LOG.info("Apache Derby repository is set to "+conf.get(CatalogConstants.CATALOG_URI));
+    LOG.info("Apache Derby repository is set to " + conf.get(CatalogConstants.CATALOG_URI));
     conf.setVar(ConfVars.CATALOG_ADDRESS, "localhost:0");
 
     catalogServer = new MiniCatalogServer(conf);
     CatalogServer catServer = catalogServer.getCatalogServer();
     InetSocketAddress sockAddr = catServer.getBindAddress();
     c.setVar(ConfVars.CATALOG_ADDRESS, NetUtils.normalizeInetSocketAddress(sockAddr));
-
+    isCatalogServerRunning = true;
     return this.catalogServer;
   }
 
@@ -323,6 +312,7 @@ public class TajoTestingCluster {
     if (catalogServer != null) {
       this.catalogServer.shutdown();
     }
+    isCatalogServerRunning = false;
   }
 
   public MiniCatalogServer getMiniCatalogCluster() {
@@ -352,10 +342,10 @@ public class TajoTestingCluster {
       c.setVar(ConfVars.ROOT_DIR,
           getMiniDFSCluster().getFileSystem().getUri() + "/tajo");
     } else {
-      c.setVar(ConfVars.ROOT_DIR, clusterTestBuildDir.getAbsolutePath() + "/tajo");
+      c.setVar(ConfVars.ROOT_DIR, testBuildDir.getAbsolutePath() + "/tajo");
     }
 
-    setupCatalogForTesting(c, clusterTestBuildDir);
+    setupCatalogForTesting(c, testBuildDir);
 
     tajoMaster = new TajoMaster();
     tajoMaster.init(c);
@@ -374,6 +364,7 @@ public class TajoTestingCluster {
     if(standbyWorkerMode) {
       startTajoWorkers(numSlaves);
     }
+    isTajoClusterRunning = true;
     LOG.info("Mini Tajo cluster is up");
     LOG.info("====================================================================================");
     LOG.info("=                           MiniTajoCluster starts up                     
        =");
@@ -473,8 +464,8 @@ public class TajoTestingCluster {
   /**
    * @throws java.io.IOException If a cluster -- dfs or engine -- already running.
    */
-  void isRunningCluster(String passedBuildPath) throws IOException {
-    if (this.clusterTestBuildDir == null || passedBuildPath != null) return;
+  void isRunningCluster() throws IOException {
+    if (!isTajoClusterRunning && !isCatalogServerRunning && !isDFSRunning)
return;
     throw new IOException("Cluster already running at " +
         this.clusterTestBuildDir);
   }
@@ -501,19 +492,13 @@ public class TajoTestingCluster {
     LOG.info("Starting up minicluster with 1 master(s) and " +
         numSlaves + " worker(s) and " + numDataNodes + " datanode(s)");
 
-    // If we already put up a cluster, fail.
-    String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
-    isRunningCluster(testBuildPath);
-    if (testBuildPath != null) {
-      LOG.info("Using passed path: " + testBuildPath);
+    // If we already bring up the cluster, fail.
+    isRunningCluster();
+    if (clusterTestBuildDir != null) {
+      LOG.info("Using passed path: " + clusterTestBuildDir);
     }
 
-    // Make a new random dir to home everything in.  Set it as system property.
-    // minidfs reads home from system property.
-    this.clusterTestBuildDir = testBuildPath == null?
-        setupClusterTestBuildDir() : new File(testBuildPath);
-
-    startMiniDFSCluster(numDataNodes, setupClusterTestBuildDir(), dataNodeHosts);
+    startMiniDFSCluster(numDataNodes, clusterTestBuildDir, dataNodeHosts);
     this.dfsCluster.waitClusterUp();
 
     hbaseUtil = new HBaseTestClusterUtil(conf, clusterTestBuildDir);
@@ -559,20 +544,11 @@ public class TajoTestingCluster {
   }
 
   public void startMiniClusterInLocal(final int numSlaves) throws Exception {
-    // If we already put up a cluster, fail.
-    String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
-    isRunningCluster(testBuildPath);
-    if (testBuildPath != null) {
-      LOG.info("Using passed path: " + testBuildPath);
-    }
-
-    // Make a new random dir to home everything in.  Set it as system property.
-    // minidfs reads home from system property.
-    this.clusterTestBuildDir = testBuildPath == null?
-        setupClusterTestBuildDir() : new File(testBuildPath);
+    isRunningCluster();
 
-    System.setProperty(TEST_DIRECTORY_KEY,
-        this.clusterTestBuildDir.getAbsolutePath());
+    if (clusterTestBuildDir != null) {
+      LOG.info("Using passed path: " + clusterTestBuildDir);
+    }
 
     startMiniTajoCluster(this.clusterTestBuildDir, numSlaves, true);
   }
@@ -592,6 +568,7 @@ public class TajoTestingCluster {
 
     if(this.catalogServer != null) {
       shutdownCatalogCluster();
+      isCatalogServerRunning = false;
     }
 
     if(this.yarnCluster != null) {
@@ -612,6 +589,7 @@ public class TajoTestingCluster {
       } catch (IOException e) {
         System.err.println("error closing file system: " + e);
       }
+      isDFSRunning = false;
     }
 
     if(this.clusterTestBuildDir != null && this.clusterTestBuildDir.exists()) {
@@ -630,6 +608,7 @@ public class TajoTestingCluster {
     }
 
     LOG.info("Minicluster is down");
+    isTajoClusterRunning = false;
   }
 
   public static TajoClient newTajoClient() throws Exception {
@@ -803,7 +782,8 @@ public class TajoTestingCluster {
       } catch (InterruptedException e) {
       }
       if (++i > 200) {
-        throw new IOException("Timed out waiting");
+        throw new IOException("Timed out waiting. expected: " + expected +
+            ", actual: " + query != null ? String.valueOf(query.getSynchronizedState()) :
String.valueOf(query));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
index aff1677..e014b52 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java
@@ -38,9 +38,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
 
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.PrintWriter;
+import java.io.*;
 import java.net.URL;
 
 import static org.junit.Assert.*;
@@ -77,7 +75,8 @@ public class TestTajoCli {
   }
 
   @After
-  public void tearDown() {
+  public void tearDown() throws IOException {
+    out.close();
     if (tajoCli != null) {
       tajoCli.close();
     }
@@ -350,38 +349,27 @@ public class TestTajoCli {
     assertOutputResult(new String(out.toByteArray()));
   }
 
-  @Test
+  @Test(timeout = 3000)
   public void testNonForwardQueryPause() throws Exception {
     final String sql = "select * from default.lineitem";
+    TajoCli cli = null;
     try {
       TableDesc tableDesc = cluster.getMaster().getCatalog().getTableDesc("default", "lineitem");
       assertNotNull(tableDesc);
       assertEquals(0L, tableDesc.getStats().getNumRows().longValue());
-      setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "2");
-      setVar(tajoCli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
-      Thread t = new Thread() {
-        public void run() {
-          try {
-            tajoCli.executeScript(sql);
-          } catch (Exception e) {
-            e.printStackTrace();
-            fail(e.getMessage());
-          }
-        }
-      };
-      t.start();
+
+      InputStream testInput = new ByteArrayInputStream(new byte[]{(byte) DefaultTajoCliOutputFormatter.QUIT_COMMAND});
+      cli = new TajoCli(cluster.getConfiguration(), new String[]{}, testInput, out);
+      setVar(cli, SessionVars.CLI_PAGE_ROWS, "2");
+      setVar(cli, SessionVars.CLI_FORMATTER_CLASS, TajoCliOutputTestFormatter.class.getName());
+
+      cli.executeScript(sql);
+
       String consoleResult;
-      while (true) {
-        Thread.sleep(3 * 1000);
-        consoleResult = new String(out.toByteArray());
-        if (consoleResult.indexOf("row") >= 0) {
-          t.interrupt();
-          break;
-        }
-      }
+      consoleResult = new String(out.toByteArray());
       assertOutputResult(consoleResult);
     } finally {
-      setVar(tajoCli, SessionVars.CLI_PAGE_ROWS, "100");
+      cli.close();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index cc7dced..0799d22 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -660,12 +660,11 @@ public class TestInsertQuery extends QueryTestCaseBase {
 
   @Test
   public final void testInsertOverwriteTableWithNonFromQuery2() throws Exception {
-    String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery");
+    String tableName = CatalogUtil.normalizeIdentifier("InsertOverwriteWithEvalQuery2");
     ResultSet res = executeString("create table " + tableName +" (col1 int4, col2 float4,
col3 text)");
     res.close();
     CatalogService catalog = testingCluster.getMaster().getCatalog();
     assertTrue(catalog.existsTable(getCurrentDatabase(), tableName));
-
     res = executeString("insert overwrite into " + tableName + " (col1, col3) select 1::INT4,
'test';");
     res.close();
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
index 42ad8da..0574bea 100644
--- a/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/querymaster/TestKillQuery.java
@@ -127,12 +127,24 @@ public class TestKillQuery {
     Query q = queryMasterTask.getQuery();
     q.handle(new QueryEvent(queryId, QueryEventType.KILL));
 
-    try{
+    try {
       cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED,
50);
-    } finally {
       assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState());
+    } catch (Exception e) {
+      e.printStackTrace();
+      if (stage != null) {
+        System.err.println(String.format("Stage: [%s] (Total: %d, Complete: %d, Success:
%d, Killed: %d, Failed: %d)",
+            stage.getId().toString(),
+            stage.getTotalScheduledObjectsCount(),
+            stage.getCompletedTaskCount(),
+            stage.getSucceededObjectCount(),
+            stage.getKilledObjectCount(),
+            stage.getFailedObjectCount()));
+      }
+      throw e;
+    } finally {
+      queryMasterTask.stop();
     }
-    queryMasterTask.stop();
   }
 
   @Test
@@ -145,6 +157,8 @@ public class TestKillQuery {
     QueryMasterTask qmt = cluster.getQueryMasterTask(queryId);
     Query query = qmt.getQuery();
 
+    // wait for a stage created
+    cluster.waitForQueryState(query, TajoProtos.QueryState.QUERY_RUNNING, 10);
     query.handle(new QueryEvent(queryId, QueryEventType.KILL));
 
     try{

http://git-wip-us.apache.org/repos/asf/tajo/blob/fd73074f/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
index e9485d0..d4ba604 100644
--- a/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testNonForwardQueryPause.result
@@ -2,4 +2,5 @@ l_orderkey,  l_partkey,  l_suppkey,  l_linenumber,  l_quantity,  l_extendedprice
 -------------------------------
 1,  1,  7706,  1,  17.0,  21168.23,  0.04,  0.02,  N,  O,  1996-03-13,  1996-02-12,  1996-03-22,
 DELIVER IN PERSON,  TRUCK,  egular courts above the
 1,  1,  7311,  2,  36.0,  45983.16,  0.09,  0.06,  N,  O,  1996-04-12,  1996-02-28,  1996-04-20,
 TAKE BACK RETURN,  MAIL,  ly final dependencies: slyly bold 
-(2 rows, continue... 'q' is quit)
\ No newline at end of file
+(2 rows, continue... 'q' is quit)
+(unknown row number, , 604 B selected)
\ No newline at end of file


Mime
View raw message