FAQ
Author: sershe
Date: Wed Apr 1 21:44:59 2015
New Revision: 1670778

URL: http://svn.apache.org/r1670778
Log:
HIVE-10103 : LLAP: Cancelling tasks fails to stop cache filling threads (Sergey Shelukhin)

Modified:
     hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
     hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
     hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java
     hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q
     hive/branches/llap/ql/src/test/results/clientpositive/orc_llap.q.out

Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java?rev=1670778&r1=1670777&r2=1670778&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java Wed Apr 1 21:44:59 2015
@@ -22,6 +22,7 @@ package org.apache.hadoop.hive.llap.io.a
  import java.io.IOException;
  import java.util.LinkedList;
  import java.util.List;
+import java.util.Random;

  import org.apache.hadoop.hive.llap.Consumer;
  import org.apache.hadoop.hive.llap.ConsumerFeedback;
@@ -126,35 +127,40 @@ public class LlapInputFormat

      @Override
      public boolean next(NullWritable key, VectorizedRowBatch value) throws IOException {
- try {
- assert value != null;
- if (isClosed) {
- throw new AssertionError("next called after close");
- }
- // Add partition cols if necessary (see VectorizedOrcInputFormat for details).
- if (isFirst) {
+ assert value != null;
+ if (isClosed) {
+ throw new AssertionError("next called after close");
+ }
+ // Add partition cols if necessary (see VectorizedOrcInputFormat for details).
+ if (isFirst) {
+ try {
            rbCtx.addPartitionColsToBatch(value);
- startRead();
- isFirst = false;
- }
- ColumnVectorBatch cvb = nextCvb();
- if (cvb == null) return false;
- int[] columnMap = rbCtx.getIncludedColumnIndexes();
- if (columnMap.length != cvb.cols.length) {
- throw new RuntimeException("Unexpected number of columns, VRB has " + columnMap.length
- + " included, but the reader returned " + cvb.cols.length);
- }
- // VRB was created from VrbCtx, so we already have pre-allocated column vectors
- for (int i = 0; i < cvb.cols.length; ++i) {
- value.cols[columnMap[i]] = cvb.cols[i]; // TODO: reuse CV objects that are replaced
+ } catch (HiveException e) {
+ throw new IOException(e);
          }
- value.selectedInUse = false;
- value.size = cvb.size;
+ startRead();
+ isFirst = false;
+ }
+ ColumnVectorBatch cvb = null;
+ try {
+ cvb = nextCvb();
        } catch (InterruptedException e) {
+ // Query might have been canceled. Stop the background processing.
+ feedback.stop();
          throw new IOException(e);
- } catch (HiveException e) {
- throw new IOException(e);
        }
+ if (cvb == null) return false;
+ int[] columnMap = rbCtx.getIncludedColumnIndexes();
+ if (columnMap.length != cvb.cols.length) {
+ throw new RuntimeException("Unexpected number of columns, VRB has " + columnMap.length
+ + " included, but the reader returned " + cvb.cols.length);
+ }
+ // VRB was created from VrbCtx, so we already have pre-allocated column vectors
+ for (int i = 0; i < cvb.cols.length; ++i) {
+ value.cols[columnMap[i]] = cvb.cols[i]; // TODO: reuse CV objects that are replaced
+ }
+ value.selectedInUse = false;
+ value.size = cvb.size;
        return true;
      }


Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java?rev=1670778&r1=1670777&r2=1670778&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java Wed Apr 1 21:44:59 2015
@@ -65,6 +65,7 @@ public class OrcEncodedDataReader extend
    private OrcFileMetadata fileMetadata;
    private Reader orcReader;
    private MetadataReader metadataReader;
+ private EncodedReader stripeReader;
    private long fileId;
    private FileSystem fs;
    /**
@@ -72,7 +73,7 @@ public class OrcEncodedDataReader extend
     * read. Contains only stripes that are read, and only columns included. null => read all RGs.
     */
    private boolean[][][] readState;
- private boolean isStopped = false, isPaused = false;
+ private volatile boolean isStopped = false, isPaused = false;

    public OrcEncodedDataReader(LowLevelCache lowLevelCache, Cache<OrcCacheKey> cache,
        OrcMetadataCache metadataCache, Configuration conf, InputSplit split,
@@ -95,8 +96,10 @@ public class OrcEncodedDataReader extend

    @Override
    public void stop() {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Encoded reader is being stopped");
+ }
      isStopped = true;
- // TODO: stop fetching if still in progress
    }

    @Override
@@ -116,7 +119,7 @@ public class OrcEncodedDataReader extend
      if (LlapIoImpl.LOGL.isInfoEnabled()) {
        LlapIoImpl.LOG.info("Processing split for " + split.getPath());
      }
- if (isStopped) return null;
+ if (processStop()) return null;
      orcReader = null;
      // 1. Get file metadata from cache, or create the reader and read it.
      // Disable filesystem caching for now; Tez closes it and FS cache will fix all that
@@ -175,19 +178,17 @@ public class OrcEncodedDataReader extend
        // Now, apply SARG if any; w/o sarg, this will just initialize readState.
        determineRgsToRead(globalIncludes, stride, stripeMetadatas);
      } catch (Throwable t) {
- cleanupReaders(null);
+ cleanupReaders();
        consumer.setError(t);
        return null;
      }

- if (isStopped) {
- cleanupReaders(null);
- return null;
- }
+ if (processStop()) return null;

      // 4. Get data from high-level cache.
- // If some cols are fully in cache, this will also give us the modified list of
- // columns to read for every stripe (null means read all of them - the usual path).
+ // If some cols are fully in cache, this will also give us the modified list of columns to
+ // read for every stripe (null means read all of them - the usual path). In any case,
+ // readState will be modified for column x rgs that were fetched from high-level cache.
      List<Integer>[] stripeColsToRead = null;
      if (cache != null) {
        try {
@@ -195,24 +196,22 @@ public class OrcEncodedDataReader extend
        } catch (Throwable t) {
          // produceDataFromCache handles its own cleanup.
          consumer.setError(t);
- cleanupReaders(null);
+ cleanupReaders();
          return null;
        }
      }
- // readState has been modified for column x rgs that were fetched from cache.

      // 5. Create encoded data reader.
      // In case if we have high-level cache, we will intercept the data and add it there;
      // otherwise just pass the data directly to the consumer.
      Consumer<EncodedColumnBatch<OrcBatchKey>> dataConsumer =
          (cache == null) ? this.consumer : this;
- EncodedReader stripeReader = null;
      try {
        ensureOrcReader();
        stripeReader = orcReader.encodedReader(fileId, lowLevelCache, dataConsumer);
      } catch (Throwable t) {
        consumer.setError(t);
- cleanupReaders(null);
+ cleanupReaders();
        return null;
      }

@@ -220,18 +219,22 @@ public class OrcEncodedDataReader extend
      // TODO: I/O threadpool could be here - one thread per stripe; for now, linear.
      OrcBatchKey stripeKey = new OrcBatchKey(fileId, -1, 0);
      for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) {
+ if (processStop()) return null;
+ int stripeIx = stripeIxFrom + stripeIxMod;
+ boolean[][] colRgs = null;
+ boolean[] stripeIncludes = null;
+ OrcStripeMetadata stripeMetadata = null;
+ StripeInformation stripe;
        try {
          List<Integer> cols = stripeColsToRead == null ? null : stripeColsToRead[stripeIxMod];
          if (cols != null && cols.isEmpty()) continue; // No need to read this stripe.
- int stripeIx = stripeIxFrom + stripeIxMod;
- StripeInformation si = fileMetadata.getStripes().get(stripeIx);
+ stripe = fileMetadata.getStripes().get(stripeIx);

          if (DebugUtils.isTraceOrcEnabled()) {
            LlapIoImpl.LOG.info("Reading stripe " + stripeIx + ": "
- + si.getOffset() + ", " + si.getLength());
+ + stripe.getOffset() + ", " + stripe.getLength());
          }
- boolean[] stripeIncludes = null;
- boolean[][] colRgs = readState[stripeIxMod];
+ colRgs = readState[stripeIxMod];

          // 6.1. Determine the columns to read (usually the same as requested).
          if (cols == null || cols.size() == colRgs.length) {
@@ -245,7 +248,6 @@ public class OrcEncodedDataReader extend
          }

          // 6.2. Ensure we have stripe metadata. We might have read it before for RG filtering.
- OrcStripeMetadata stripeMetadata;
          if (stripeMetadatas != null) {
            stripeMetadata = stripeMetadatas.get(stripeIxMod);
          } else {
@@ -254,7 +256,7 @@ public class OrcEncodedDataReader extend
            if (stripeMetadata == null) {
              ensureMetadataReader();
              stripeMetadata = metadataCache.putStripeMetadata(new OrcStripeMetadata(
- stripeKey, metadataReader, si, stripeIncludes, sargColumns));
+ stripeKey, metadataReader, stripe, stripeIncludes, sargColumns));
              if (DebugUtils.isTraceOrcEnabled()) {
                LlapIoImpl.LOG.info("Caching stripe " + stripeKey.stripeIx
                    + " metadata with includes: " + DebugUtils.toString(stripeIncludes));
@@ -269,17 +271,27 @@ public class OrcEncodedDataReader extend
                  + " metadata for includes: " + DebugUtils.toString(stripeIncludes));
            }
            ensureMetadataReader();
- updateLoadedIndexes(stripeMetadata, si, stripeIncludes, sargColumns);
+ updateLoadedIndexes(stripeMetadata, stripe, stripeIncludes, sargColumns);
          }
- // 6.3. Finally, hand off to the stripe reader to produce the data.
- // This is a sync call that will feed data to the consumer.
+ } catch (Throwable t) {
+ consumer.setError(t);
+ cleanupReaders();
+ return null;
+ }
+ if (processStop()) return null;
+
+ // 6.3. Finally, hand off to the stripe reader to produce the data.
+ // This is a sync call that will feed data to the consumer.
+ try {
          // TODO: readEncodedColumns is not supposed to throw; errors should be propagated thru
          // consumer. It is potentially holding locked buffers, and must perform its own cleanup.
- stripeReader.readEncodedColumns(stripeIx, si, stripeMetadata.getRowIndexes(),
+ // Also, currently readEncodedColumns is not stoppable. The consumer will discard the
+ // data it receives for one stripe. We could probably interrupt it, if it checked that.
+ stripeReader.readEncodedColumns(stripeIx, stripe, stripeMetadata.getRowIndexes(),
              stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes, colRgs);
        } catch (Throwable t) {
          consumer.setError(t);
- cleanupReaders(stripeReader);
+ cleanupReaders();
          return null;
        }
      }
@@ -291,10 +303,17 @@ public class OrcEncodedDataReader extend
      }

      // Close the stripe reader, we are done reading.
- stripeReader.close();
+ cleanupReaders();
      return null;
    }

+ private boolean processStop() {
+ if (!isStopped) return false;
+ LOG.info("Encoded data reader is stopping");
+ cleanupReaders();
+ return true;
+ }
+
    private static long determineFileId(FileSystem fs, FileSplit split) throws IOException {
      if (split instanceof OrcSplit) {
        Long fileId = ((OrcSplit)split).getFileId();
@@ -344,7 +363,7 @@ public class OrcEncodedDataReader extend
    /**
     * Closes the stripe readers (on error).
     */
- private void cleanupReaders(EncodedReader er) {
+ private void cleanupReaders() {
      if (metadataReader != null) {
        try {
          metadataReader.close();
@@ -352,9 +371,9 @@ public class OrcEncodedDataReader extend
          // Ignore.
        }
      }
- if (er != null) {
+ if (stripeReader != null) {
        try {
- er.close();
+ stripeReader.close();
        } catch (IOException ex) {
          // Ignore.
        }

Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java?rev=1670778&r1=1670777&r2=1670778&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordSource.java Wed Apr 1 21:44:59 2015
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.ql.metadat
  import org.apache.hadoop.io.Writable;
  import org.apache.hadoop.mapred.JobConf;
  import org.apache.hadoop.util.StringUtils;
+import org.apache.tez.mapreduce.lib.MRReader;
  import org.apache.tez.runtime.library.api.KeyValueReader;

  /**
@@ -63,11 +64,13 @@ public class MapRecordSource implements
          try {
            value = reader.getCurrentValue();
          } catch (IOException e) {
+ closeReader();
            throw new HiveException(e);
          }
          return processRow(value);
        }
      } catch (IOException e) {
+ closeReader();
        throw new HiveException(e);
      }
      return false;
@@ -88,10 +91,25 @@ public class MapRecordSource implements
          throw (OutOfMemoryError) e;
        } else {
          LOG.fatal(StringUtils.stringifyException(e));
+ closeReader();
          throw new RuntimeException(e);
        }
      }
      return true; // give me more
    }

+ private void closeReader() {
+ if (!(reader instanceof MRReader)) {
+ LOG.warn("Cannot close " + (reader == null ? null : reader.getClass()));
+ return;
+ }
+ LOG.info("Closing MRReader on error");
+ MRReader mrReader = (MRReader)reader;
+ try {
+ mrReader.close();
+ } catch (IOException ex) {
+ LOG.error("Failed to close the reader; ignoring", ex);
+ }
+ }
+
  }

Modified: hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q?rev=1670778&r1=1670777&r2=1670778&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientpositive/orc_llap.q Wed Apr 1 21:44:59 2015
@@ -5,7 +5,7 @@ SET hive.llap.io.enabled=false;
  SET hive.exec.orc.default.buffer.size=32768;
  SET hive.exec.orc.default.row.index.stride=1000;
  SET hive.optimize.index.filter=true;
-set hive.auto.convert.join=true;
+set hive.auto.convert.join=false;

  CREATE TABLE orc_llap(
      ctinyint TINYINT,
@@ -33,6 +33,7 @@ select ctinyint + i, csmallint + i, cint
  from alltypesorc cross join cross_numbers;

  SET hive.llap.io.enabled=true;
+set hive.auto.convert.join=true;


  -- Hash cannot be vectorized, so run hash as the last step on a temp table

Modified: hive/branches/llap/ql/src/test/results/clientpositive/orc_llap.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/results/clientpositive/orc_llap.q.out?rev=1670778&r1=1670777&r2=1670778&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/results/clientpositive/orc_llap.q.out (original)
+++ hive/branches/llap/ql/src/test/results/clientpositive/orc_llap.q.out Wed Apr 1 21:44:59 2015
@@ -53,7 +53,7 @@ POSTHOOK: type: QUERY
  POSTHOOK: Input: default@alltypesorc
  POSTHOOK: Output: default@cross_numbers
  POSTHOOK: Lineage: cross_numbers.i EXPRESSION [(alltypesorc)alltypesorc.FieldSchema(name:csmallint, type:smallint, comment:null), ]
-Warning: Map Join MAPJOIN[12][bigTable=?] in task 'Stage-4:MAPRED' is a cross product
+Warning: Shuffle Join JOIN[7][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
  PREHOOK: query: insert into table orc_llap
  select ctinyint + i, csmallint + i, cint + i, cbigint + i, cfloat + i, cdouble + i, cstring1, cstring2, ctimestamp1, ctimestamp2, cboolean1, cboolean2
  from alltypesorc cross join cross_numbers
@@ -536,17 +536,17 @@ STAGE PLANS:
            TableScan
              alias: orc_llap
              filterExpr: ((cint > 10) and cbigint is not null) (type: boolean)
- Statistics: Num rows: 99864 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 99580 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Filter Operator
                predicate: ((cint > 10) and cbigint is not null) (type: boolean)
- Statistics: Num rows: 16644 Data size: 266304 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16597 Data size: 265554 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: cint (type: int), csmallint (type: smallint), cbigint (type: bigint)
                  outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 16644 Data size: 266304 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16597 Data size: 265554 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
- Statistics: Num rows: 16644 Data size: 266304 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 16597 Data size: 265554 Basic stats: COMPLETE Column stats: NONE
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat
                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -606,17 +606,17 @@ STAGE PLANS:
            TableScan
              alias: orc_llap
              filterExpr: ((cint > 10) and cbigint is not null) (type: boolean)
- Statistics: Num rows: 4993 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4979 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Filter Operator
                predicate: ((cint > 10) and cbigint is not null) (type: boolean)
- Statistics: Num rows: 832 Data size: 266251 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 830 Data size: 265602 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 832 Data size: 266251 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 830 Data size: 265602 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
- Statistics: Num rows: 832 Data size: 266251 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 830 Data size: 265602 Basic stats: COMPLETE Column stats: NONE
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat
                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -676,17 +676,17 @@ STAGE PLANS:
            TableScan
              alias: orc_llap
              filterExpr: ((cint > 5) and (cint < 10)) (type: boolean)
- Statistics: Num rows: 15363 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 15320 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Filter Operator
                predicate: ((cint > 5) and (cint < 10)) (type: boolean)
- Statistics: Num rows: 1707 Data size: 177536 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1702 Data size: 177009 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: cstring2 (type: string)
                  outputColumnNames: _col0
- Statistics: Num rows: 1707 Data size: 177536 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1702 Data size: 177009 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
- Statistics: Num rows: 1707 Data size: 177536 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1702 Data size: 177009 Basic stats: COMPLETE Column stats: NONE
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat
                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -745,22 +745,22 @@ STAGE PLANS:
        Map Operator Tree:
            TableScan
              alias: orc_llap
- Statistics: Num rows: 7989 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7966 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Select Operator
                expressions: cstring1 (type: string), cstring2 (type: string)
                outputColumnNames: _col0, _col1
- Statistics: Num rows: 7989 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7966 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
                Group By Operator
                  aggregations: count()
                  keys: _col0 (type: string), _col1 (type: string)
                  mode: hash
                  outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 7989 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7966 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
                  Reduce Output Operator
                    key expressions: _col0 (type: string), _col1 (type: string)
                    sort order: ++
                    Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
- Statistics: Num rows: 7989 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 7966 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
                    value expressions: _col2 (type: bigint)
        Execution mode: vectorized
        LLAP IO: all inputs
@@ -770,10 +770,10 @@ STAGE PLANS:
            keys: KEY._col0 (type: string), KEY._col1 (type: string)
            mode: mergepartial
            outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3994 Data size: 798813 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3983 Data size: 796646 Basic stats: COMPLETE Column stats: NONE
            File Output Operator
              compressed: false
- Statistics: Num rows: 3994 Data size: 798813 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3983 Data size: 796646 Basic stats: COMPLETE Column stats: NONE
              table:
                  input format: org.apache.hadoop.mapred.TextInputFormat
                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -837,10 +837,10 @@ STAGE PLANS:
            TableScan
              alias: o1
              filterExpr: (csmallint is not null and cbigint is not null) (type: boolean)
- Statistics: Num rows: 14266 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14225 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Filter Operator
                predicate: (csmallint is not null and cbigint is not null) (type: boolean)
- Statistics: Num rows: 3567 Data size: 399513 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3557 Data size: 398407 Basic stats: COMPLETE Column stats: NONE
                HashTable Sink Operator
                  keys:
                    0 csmallint (type: smallint)
@@ -852,10 +852,10 @@ STAGE PLANS:
            TableScan
              alias: o2
              filterExpr: (csmallint is not null and cbigint is not null) (type: boolean)
- Statistics: Num rows: 14266 Data size: 1597828 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 14225 Data size: 1593293 Basic stats: COMPLETE Column stats: NONE
              Filter Operator
                predicate: (csmallint is not null and cbigint is not null) (type: boolean)
- Statistics: Num rows: 3567 Data size: 399513 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3557 Data size: 398407 Basic stats: COMPLETE Column stats: NONE
                Map Join Operator
                  condition map:
                       Inner Join 0 to 1
@@ -863,14 +863,14 @@ STAGE PLANS:
                    0 csmallint (type: smallint)
                    1 csmallint (type: smallint)
                  outputColumnNames: _col6, _col22
- Statistics: Num rows: 3923 Data size: 439464 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3912 Data size: 438247 Basic stats: COMPLETE Column stats: NONE
                  Select Operator
                    expressions: _col6 (type: string), _col22 (type: string)
                    outputColumnNames: _col0, _col1
- Statistics: Num rows: 3923 Data size: 439464 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3912 Data size: 438247 Basic stats: COMPLETE Column stats: NONE
                    File Output Operator
                      compressed: false
- Statistics: Num rows: 3923 Data size: 439464 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3912 Data size: 438247 Basic stats: COMPLETE Column stats: NONE
                      table:
                          input format: org.apache.hadoop.mapred.TextInputFormat
                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 1, '15 at 9:45p
activeApr 1, '15 at 9:45p
posts1
users1
websitehive.apache.org

1 user in discussion

Sershe: 1 post

People

Translate

site design / logo © 2022 Grokbase