FAQ
Author: sershe
Date: Mon Oct 27 22:59:12 2014
New Revision: 1634725

URL: http://svn.apache.org/r1634725
Log:
HIVE-8021 : CBO: support CTAS and insert ... select (Sergey Shelukhin, reviewed by Laljo John Pullokkaran)

Added:
     hive/trunk/ql/src/test/queries/clientpositive/insert0.q
     hive/trunk/ql/src/test/results/clientpositive/insert0.q.out
Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
     hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q
     hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q
     hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q
     hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
     hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/index/RewriteParseContextGenerator.java Mon Oct 27 22:59:12 2014
@@ -112,7 +112,7 @@ public final class RewriteParseContextGe
      ((SemanticAnalyzer) sem).initParseCtx(subPCtx);

      LOG.info("Starting Sub-query Semantic Analysis");
- sem.doPhase1(child, qb, sem.initPhase1Ctx());
+ sem.doPhase1(child, qb, sem.initPhase1Ctx(), null);
      LOG.info("Completed phase 1 of Sub-query Semantic Analysis");

      sem.getMetaData(qb);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Oct 27 22:59:12 2014
@@ -79,7 +79,7 @@ import com.google.common.annotations.Vis
   *
   */
  public abstract class BaseSemanticAnalyzer {
- private static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
+ protected static final Log STATIC_LOG = LogFactory.getLog(BaseSemanticAnalyzer.class.getName());
    protected final Hive db;
    protected final HiveConf conf;
    protected List<Task<? extends Serializable>> rootTasks;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon Oct 27 22:59:12 2014
@@ -33,6 +33,7 @@ import java.util.HashMap;
  import java.util.HashSet;
  import java.util.Iterator;
  import java.util.LinkedHashMap;
+import java.util.LinkedList;
  import java.util.List;
  import java.util.Map;
  import java.util.Map.Entry;
@@ -368,7 +369,7 @@ public class SemanticAnalyzer extends Ba
    //flag for partial scan during analyze ... compute statistics
    protected boolean partialscan;

- private volatile boolean runCBO = true;
+ private volatile boolean runCBO = true; // TODO: why is this volatile?
    private volatile boolean disableJoinMerge = false;

    /*
@@ -380,6 +381,9 @@ public class SemanticAnalyzer extends Ba
     */
    private ArrayList<String> ctesExpanded;

+ /** Not thread-safe. */
+ private ASTSearcher astSearcher = new ASTSearcher();
+
    private static class Phase1Ctx {
      String dest;
      int nextNum;
@@ -512,7 +516,7 @@ public class SemanticAnalyzer extends Ba
      case HiveParser.TOK_QUERY: {
        QB qb = new QB(id, alias, true);
        Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(ast, qb, ctx_1);
+ doPhase1(ast, qb, ctx_1, null);

        qbexpr.setOpcode(QBExpr.Opcode.NULLOP);
        qbexpr.setQB(qb);
@@ -1177,6 +1181,30 @@ public class SemanticAnalyzer extends Ba
      return alias;
    }

+ /** The context that doPhase1 uses to populate information pertaining
+ * to CBO (currently, this is used for CTAS and insert-as-select). */
+ private static class PreCboCtx {
+ enum Type {
+ NONE,
+ INSERT,
+ CTAS,
+
+ UNEXPECTED
+ }
+ public ASTNode nodeOfInterest;
+ public Type type = Type.NONE;
+ public void set(Type type, ASTNode ast) {
+ if (this.type != Type.NONE) {
+ STATIC_LOG.warn("Setting " + type + " when already " + this.type
+ + "; node " + ast.dump() + " vs old node " + nodeOfInterest.dump());
+ this.type = Type.UNEXPECTED;
+ return;
+ }
+ this.type = type;
+ this.nodeOfInterest = ast;
+ }
+ }
+
    /**
     * Phase 1: (including, but not limited to):
     *
@@ -1194,7 +1222,7 @@ public class SemanticAnalyzer extends Ba
     * @throws SemanticException
     */
    @SuppressWarnings({"fallthrough", "nls"})
- public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1)
+ public boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PreCboCtx cboCtx)
        throws SemanticException {

      boolean phase1Result = true;
@@ -1238,20 +1266,26 @@ public class SemanticAnalyzer extends Ba
          String currentDatabase = SessionState.get().getCurrentDatabase();
          String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
          qbp.addInsertIntoTable(tab_name);
- // TODO: is this supposed to fall thru?

        case HiveParser.TOK_DESTINATION:
          ctx_1.dest = "insclause-" + ctx_1.nextNum;
          ctx_1.nextNum++;
+ boolean isTmpFileDest = false;
+ if (ast.getChildCount() > 0 && ast.getChild(0) instanceof ASTNode) {
+ ASTNode ch = (ASTNode)ast.getChild(0);
+ if (ch.getToken().getType() == HiveParser.TOK_DIR
+ && ch.getChildCount() > 0 && ch.getChild(0) instanceof ASTNode) {
+ ch = (ASTNode)ch.getChild(0);
+ isTmpFileDest = ch.getToken().getType() == HiveParser.TOK_TMP_FILE;
+ }
+ }

          // is there a insert in the subquery
- if (qbp.getIsSubQ()) {
- ASTNode ch = (ASTNode) ast.getChild(0);
- if ((ch.getToken().getType() != HiveParser.TOK_DIR)
- || (((ASTNode) ch.getChild(0)).getToken().getType() != HiveParser.TOK_TMP_FILE)) {
- throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY
- .getMsg(ast));
- }
+ if (qbp.getIsSubQ() && !isTmpFileDest) {
+ throw new SemanticException(ErrorMsg.NO_INSERT_INSUBQUERY.getMsg(ast));
+ }
+ if (cboCtx != null && !isTmpFileDest) {
+ cboCtx.set(PreCboCtx.Type.INSERT, ast);
          }
          qbp.setDestForClause(ctx_1.dest, (ASTNode) ast.getChild(0));

@@ -1480,12 +1514,18 @@ public class SemanticAnalyzer extends Ba
        int child_count = ast.getChildCount();
        for (int child_pos = 0; child_pos < child_count && phase1Result; ++child_pos) {
          // Recurse
- phase1Result = phase1Result && doPhase1((ASTNode) ast.getChild(child_pos), qb, ctx_1);
+ phase1Result = phase1Result && doPhase1(
+ (ASTNode)ast.getChild(child_pos), qb, ctx_1, cboCtx);
        }
      }
      return phase1Result;
    }

+ private void traceLogAst(ASTNode ast, String what) {
+ if (!LOG.isTraceEnabled()) return;
+ LOG.trace(what + ast.dump());
+ }
+
    private void getMetaData(QBExpr qbexpr, ReadEntity parentInput)
        throws SemanticException {
      if (qbexpr.getOpcode() == QBExpr.Opcode.NULLOP) {
@@ -1760,6 +1800,7 @@ public class SemanticAnalyzer extends Ba
                  qb.getParseInfo().addTableSpec(ts.tableName.toLowerCase(), ts);
                }
              } else {
+ // This is the only place where isQuery is set to true; it defaults to false.
                qb.setIsQuery(true);
                fname = ctx.getMRTmpPath().toString();
                ctx.setResDir(new Path(fname));
@@ -2464,7 +2505,7 @@ public class SemanticAnalyzer extends Ba
        ISubQueryJoinInfo subQueryPredicate) throws SemanticException {
      qbSQ.setSubQueryDef(subQueryPredicate.getSubQuery());
      Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1);
+ doPhase1(subQueryPredicate.getSubQueryAST(), qbSQ, ctx_1, null);
      getMetaData(qbSQ);
      Operator op = genPlan(qbSQ);
      return op;
@@ -6197,8 +6238,7 @@ public class SemanticAnalyzer extends Ba
        ArrayList<ColumnInfo> colInfos = inputRR.getColumnInfos();

        // CTAS case: the file output format and serde are defined by the create
- // table command
- // rather than taking the default value
+ // table command rather than taking the default value
        List<FieldSchema> field_schemas = null;
        CreateTableDesc tblDesc = qb.getTableDesc();
        if (tblDesc != null) {
@@ -6219,7 +6259,8 @@ public class SemanticAnalyzer extends Ba
            if (!("".equals(nm[0])) && nm[1] != null) {
              colName = unescapeIdentifier(colInfo.getAlias()).toLowerCase(); // remove ``
            }
- col.setName(colName);;
+ String ctasColName = fixCtasColumnName(colName, colInfo, inputRR);
+ col.setName(ctasColName);
            col.setType(colInfo.getType().getTypeName());
            field_schemas.add(col);
          }
@@ -6397,6 +6438,14 @@ public class SemanticAnalyzer extends Ba
      return output;
    }

+ private static String fixCtasColumnName(String colName, ColumnInfo colInfo, RowResolver rr) {
+ int lastDot = colName.lastIndexOf('.');
+ if (lastDot < 0) return colName; // alias is not fully qualified
+ String nqColumnName = colName.substring(lastDot + 1);
+ STATIC_LOG.debug("Replacing " + colName + " (produced by CBO) by " + nqColumnName);
+ return nqColumnName;
+ }
+
    // Check constraints on acid tables. This includes
    // * no insert overwrites
    // * no use of vectorization
@@ -9875,11 +9924,14 @@ public class SemanticAnalyzer extends Ba

      // analyze and process the position alias
      processPositionAlias(ast);
+ // Check configuration for CBO first.
+ runCBO = runCBO && HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED);

      // analyze create table command
+ PreCboCtx cboCtx = runCBO ? new PreCboCtx() : null;
      if (ast.getToken().getType() == HiveParser.TOK_CREATETABLE) {
        // if it is not CTAS, we don't need to go further and just return
- if ((child = analyzeCreateTable(ast, qb)) == null) {
+ if ((child = analyzeCreateTable(ast, qb, cboCtx)) == null) {
          return;
        }
      } else {
@@ -9888,7 +9940,8 @@ public class SemanticAnalyzer extends Ba

      // analyze create view command
      if (ast.getToken().getType() == HiveParser.TOK_CREATEVIEW ||
- (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
+ (ast.getToken().getType() == HiveParser.TOK_ALTERVIEW
+ && ast.getChild(1).getType() == HiveParser.TOK_QUERY)) {
        child = analyzeCreateView(ast, qb);
        SessionState.get().setCommandType(HiveOperation.CREATEVIEW);
        if (child == null) {
@@ -9901,7 +9954,7 @@ public class SemanticAnalyzer extends Ba

      // continue analyzing from the child ASTNode.
      Phase1Ctx ctx_1 = initPhase1Ctx();
- if (!doPhase1(child, qb, ctx_1)) {
+ if (!doPhase1(child, qb, ctx_1, cboCtx)) {
        // if phase1Result false return
        return;
      }
@@ -9911,19 +9964,16 @@ public class SemanticAnalyzer extends Ba
      getMetaData(qb);
      LOG.info("Completed getting MetaData in Semantic Analysis");

-
+ // Note: for now, we don't actually pass the queryForCbo to CBO, because it accepts qb, not
+ // AST, and can also access all the private stuff in SA. We rely on the fact that CBO
+ // ignores the unknown tokens (create table, destination), so if the query is otherwise ok,
+ // it is as if we did remove those and gave CBO the proper AST. That is kinda hacky.
      if (runCBO) {
- boolean tokenTypeIsQuery = ast.getToken().getType() == HiveParser.TOK_QUERY
- || ast.getToken().getType() == HiveParser.TOK_EXPLAIN;
- if (!tokenTypeIsQuery || createVwDesc != null
- || !HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_CBO_ENABLED)
- || !canHandleQuery(qb, true) || !HiveOptiqUtil.validateASTForCBO(ast)) {
- runCBO = false;
- }
-
- if (runCBO) {
- disableJoinMerge = true;
+ ASTNode queryForCbo = ast;
+ if (cboCtx.type == PreCboCtx.Type.CTAS) {
+ queryForCbo = cboCtx.nodeOfInterest; // nodeOfInterest is the query
        }
+ runCBO = canHandleAstForCbo(queryForCbo, qb, cboCtx);
      }

      // Save the result schema derived from the sink operator produced
@@ -9933,6 +9983,7 @@ public class SemanticAnalyzer extends Ba
      Operator sinkOp = null;

      if (runCBO) {
+ disableJoinMerge = true;
        OptiqBasedPlanner optiqPlanner = new OptiqBasedPlanner();
        boolean reAnalyzeAST = false;

@@ -9940,10 +9991,17 @@ public class SemanticAnalyzer extends Ba
          // 1. Gen Optimized AST
          ASTNode newAST = optiqPlanner.getOptimizedAST(prunedPartitions);

+ // 1.1. Fix up the query for insert/ctas
+ newAST = fixUpCtasAndInsertAfterCbo(ast, newAST, cboCtx);
+
          // 2. Regen OP plan from optimized AST
          init(false);
+ if (cboCtx.type == PreCboCtx.Type.CTAS) {
+ // Redo create-table analysis, because it's not part of doPhase1.
+ newAST = reAnalyzeCtasAfterCbo(newAST);
+ }
          ctx_1 = initPhase1Ctx();
- if (!doPhase1(newAST, qb, ctx_1)) {
+ if (!doPhase1(newAST, qb, ctx_1, null)) {
            throw new RuntimeException(
                "Couldn't do phase1 on CBO optimized query plan");
          }
@@ -10089,6 +10147,118 @@ public class SemanticAnalyzer extends Ba
      return;
    }

+ private ASTNode fixUpCtasAndInsertAfterCbo(
+ ASTNode originalAst, ASTNode newAst, PreCboCtx cboCtx) throws SemanticException {
+ switch (cboCtx.type) {
+ case NONE: return newAst; // nothing to do
+ case CTAS: {
+ // Patch the optimized query back into original CTAS AST, replacing the original query.
+ replaceASTChild(cboCtx.nodeOfInterest, newAst);
+ return originalAst;
+ }
+ case INSERT: {
+ // We need to patch the dest back to original into new query.
+ // This makes assumptions about the structure of the AST.
+ ASTNode newDest = astSearcher.simpleBreadthFirstSearch(
+ newAst, HiveParser.TOK_QUERY, HiveParser.TOK_INSERT, HiveParser.TOK_DESTINATION);
+ if (newDest == null) {
+ LOG.error("Cannot find destination after CBO; new ast is "+ newAst.dump());
+ throw new SemanticException("Cannot find destination after CBO");
+ }
+ replaceASTChild(newDest, cboCtx.nodeOfInterest);
+ return newAst;
+ }
+ default: throw new AssertionError("Unexpected type " + cboCtx.type);
+ }
+ }
+
+ private ASTNode reAnalyzeCtasAfterCbo(ASTNode newAst) throws SemanticException {
+ // analyzeCreateTable uses this.ast, but doPhase1 doesn't, so only reset it here.
+ this.ast = newAst;
+ newAst = analyzeCreateTable(newAst, qb, null);
+ if (newAst == null) {
+ LOG.error("analyzeCreateTable failed to initialize CTAS after CBO;"
+ + " new ast is " + this.ast.dump());
+ throw new SemanticException("analyzeCreateTable failed to initialize CTAS after CBO");
+ }
+ return newAst;
+ }
+
+ private boolean canHandleAstForCbo(ASTNode ast, QB qb, PreCboCtx cboCtx) {
+ int root = ast.getToken().getType();
+ boolean needToLogMessage = LOG.isInfoEnabled();
+ boolean isSupportedRoot =
+ root == HiveParser.TOK_QUERY || root == HiveParser.TOK_EXPLAIN || qb.isCTAS();
+ // Check AST.
+ // Assumption: If top level QB is query then everything below it must also be Query
+ // Can there be an insert or CTAS that wouldn't
+ // be supported and would require additional checks similar to IsQuery?
+ boolean isSupportedType =
+ qb.getIsQuery() || qb.isCTAS() || cboCtx.type == PreCboCtx.Type.INSERT;
+ boolean result = isSupportedRoot && isSupportedType && createVwDesc == null;
+ if (!result) {
+ if (needToLogMessage) {
+ String msg = "";
+ if (!isSupportedRoot) msg += "doesn't have QUERY or EXPLAIN as root and not a CTAS; ";
+ if (!isSupportedType) msg += "is not a query, CTAS, or insert; ";
+ if (createVwDesc != null) msg += "has create view; ";
+
+ if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+ LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+ }
+ return false;
+ }
+ // Now check QB in more detail. canHandleQbForCbo returns null if query can be handled.
+ String msg = canHandleQbForCbo(qb, true, needToLogMessage);
+ if (msg == null) {
+ return true;
+ }
+ if (needToLogMessage) {
+ LOG.info("Not invoking CBO because the statement " + msg.substring(0, msg.length() - 2));
+ }
+ return false;
+ }
+
+ private class ASTSearcher {
+ private final LinkedList<ASTNode> searchQueue = new LinkedList<ASTNode>();
+ /**
+ * Performs breadth-first search of the AST for a nested set of tokens. Tokens don't have to be
+ * each others' direct children, they can be separated by layers of other tokens. For each token
+ * in the list, the first one found is matched and there's no backtracking; thus, if AST has
+ * multiple instances of some token, of which only one matches, it is not guaranteed to be found.
+ * We use this for simple things.
+ * Not thread-safe - reuses searchQueue.
+ */
+ public ASTNode simpleBreadthFirstSearch(ASTNode ast, int... tokens) {
+ searchQueue.clear();
+ searchQueue.add(ast);
+ for (int i = 0; i < tokens.length; ++i) {
+ boolean found = false;
+ int token = tokens[i];
+ while (!searchQueue.isEmpty() && !found) {
+ ASTNode next = searchQueue.poll();
+ found = next.getType() == token;
+ if (found) {
+ if (i == tokens.length - 1) return next;
+ searchQueue.clear();
+ }
+ for (int j = 0; j < next.getChildCount(); ++j) {
+ searchQueue.add((ASTNode)next.getChild(j));
+ }
+ }
+ if (!found) return null;
+ }
+ return null;
+ }
+ }
+
+ private void replaceASTChild(ASTNode child, ASTNode newChild) {
+ ASTNode parent = (ASTNode)child.parent;
+ int childIndex = child.childIndex;
+ parent.deleteChild(childIndex);
+ parent.insertChild(childIndex, newChild);
+ }
+
    private void putAccessedColumnsToReadEntity(HashSet<ReadEntity> inputs, ColumnAccessInfo columnAccessInfo) {
      Map<String, List<String>> tableToColumnAccessMap = columnAccessInfo.getTableToColumnAccessMap();
      if (tableToColumnAccessMap != null && !tableToColumnAccessMap.isEmpty()) {
@@ -10585,8 +10755,8 @@ public class SemanticAnalyzer extends Ba
     * the semantic analyzer need to deal with the select statement with respect
     * to the SerDe and Storage Format.
     */
- private ASTNode analyzeCreateTable(ASTNode ast, QB qb)
- throws SemanticException {
+ private ASTNode analyzeCreateTable(
+ ASTNode ast, QB qb, PreCboCtx cboCtx) throws SemanticException {
      String[] qualifiedTabName = getQualifiedTableName((ASTNode) ast.getChild(0));
      String dbDotTab = getDotName(qualifiedTabName);

@@ -10676,6 +10846,9 @@ public class SemanticAnalyzer extends Ba
            throw new SemanticException(ErrorMsg.CTAS_EXTTBL_COEXISTENCE.getMsg());
          }
          command_type = CTAS;
+ if (cboCtx != null) {
+ cboCtx.set(PreCboCtx.Type.CTAS, child);
+ }
          selectStmt = child;
          break;
        case HiveParser.TOK_TABCOLLIST:
@@ -12247,27 +12420,43 @@ public class SemanticAnalyzer extends Ba

    /**** Temporary Place Holder For Optiq plan Gen, Optimizer ****/

- /*
- * Entry point to Optimizations using Optiq.
+ /**
+ * Entry point to Optimizations using Optiq. Checks whether Optiq can handle the query.
+ * @param qbToChk Query block to check.
+ * @param verbose Whether return value should be verbose in case of failure.
+ * @return null if the query can be handled; non-null reason string if it cannot be.
     */
- private boolean canHandleQuery(QB qbToChk, boolean topLevelQB) {
- boolean runOptiqPlanner = false;
+ private String canHandleQbForCbo(QB qbToChk, boolean topLevelQB, boolean verbose) {
      // Assumption:
      // 1. If top level QB is query then everything below it must also be Query
      // 2. Nested Subquery will return false for qbToChk.getIsQuery()
- if ((!topLevelQB || qbToChk.getIsQuery())
- && (!conf.getBoolVar(ConfVars.HIVE_IN_TEST) || conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict"))
- && (!topLevelQB || (queryProperties.getJoinCount() > 1) || conf.getBoolVar(ConfVars.HIVE_IN_TEST))
- && !queryProperties.hasClusterBy() && !queryProperties.hasDistributeBy()
- && !queryProperties.hasSortBy() && !queryProperties.hasPTF()
- && !queryProperties.usesScript() && !queryProperties.hasMultiDestQuery()
- && !queryProperties.hasLateralViews()) {
- runOptiqPlanner = true;
- } else {
- LOG.info("Can not invoke CBO; query contains operators not supported for CBO.");
- }
+ boolean isInTest = conf.getBoolVar(ConfVars.HIVE_IN_TEST);
+ boolean isStrictTest = isInTest
+ && !conf.getVar(ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("nonstrict");
+ boolean hasEnoughJoins = !topLevelQB || (queryProperties.getJoinCount() > 1) || isInTest;
+ if (!isStrictTest && hasEnoughJoins && !queryProperties.hasClusterBy()
+ && !queryProperties.hasDistributeBy() && !queryProperties.hasSortBy()
+ && !queryProperties.hasPTF() && !queryProperties.usesScript()
+ && !queryProperties.hasMultiDestQuery() && !queryProperties.hasLateralViews()) {
+ return null; // Ok to run CBO.
+ }
+
+ // Not ok to run CBO, build error message.
+ String msg = "";
+ if (verbose) {
+ if (isStrictTest) msg += "is in test running in mode other than nonstrict; ";
+ if (!hasEnoughJoins) msg += "has too few joins; ";
+ if (queryProperties.hasClusterBy()) msg += "has cluster by; ";
+ if (queryProperties.hasDistributeBy()) msg += "has distribute by; ";
+ if (queryProperties.hasSortBy()) msg += "has sort by; ";
+ if (queryProperties.hasPTF()) msg += "has PTF; ";
+ if (queryProperties.usesScript()) msg += "uses scripts; ";
+ if (queryProperties.hasMultiDestQuery()) msg += "is a multi-destination query; ";
+ if (queryProperties.hasLateralViews()) msg += "has lateral views; ";

- return runOptiqPlanner;
+ if (msg.isEmpty()) msg += "has some unspecified limitations; ";
+ }
+ return msg;
    }

    private class OptiqBasedPlanner implements Frameworks.PlannerAction<RelNode> {
@@ -12984,7 +13173,7 @@ public class SemanticAnalyzer extends Ba
            QB qbSQ = new QB(subQuery.getOuterQueryId(), subQuery.getAlias(), true);
            qbSQ.setSubQueryDef(subQuery.getSubQuery());
            Phase1Ctx ctx_1 = initPhase1Ctx();
- doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1);
+ doPhase1(subQuery.getSubQueryAST(), qbSQ, ctx_1, null);
            getMetaData(qbSQ);
            RelNode subQueryRelNode = genLogicalPlan(qbSQ, false);
            aliasToRel.put(subQuery.getAlias(), subQueryRelNode);
@@ -13012,7 +13201,7 @@ public class SemanticAnalyzer extends Ba
              QB qbSQ_nic = new QB(subQuery.getOuterQueryId(), notInCheck.getAlias(), true);
              qbSQ_nic.setSubQueryDef(notInCheck.getSubQuery());
              ctx_1 = initPhase1Ctx();
- doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1);
+ doPhase1(notInCheck.getSubQueryAST(), qbSQ_nic, ctx_1, null);
              getMetaData(qbSQ_nic);
              RelNode subQueryNICRelNode = genLogicalPlan(qbSQ_nic, false);
              aliasToRel.put(notInCheck.getAlias(), subQueryNICRelNode);
@@ -14069,11 +14258,14 @@ public class SemanticAnalyzer extends Ba
        // First generate all the opInfos for the elements in the from clause
        Map<String, RelNode> aliasToRel = new HashMap<String, RelNode>();

- // 0. Check if we can handle the query
- // This check is needed here because of SubQuery
- if (!canHandleQuery(qb, false)) {
- String msg = String.format("CBO Can not handle Sub Query");
- LOG.debug(msg);
+ // 0. Check if we can handle the SubQuery;
+ // canHandleQbForCbo returns null if the query can be handled.
+ String reason = canHandleQbForCbo(qb, false, LOG.isDebugEnabled());
+ if (reason != null) {
+ String msg = "CBO can not handle Sub Query";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(msg + " because it: " + reason);
+ }
          throw new OptiqSemanticException(msg);
        }


Modified: hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/cbo_correctness.q Mon Oct 27 22:59:12 2014
@@ -485,4 +485,4 @@ select unionsrc.key, count(1) FROM (sele
  -- Windowing
  select *, rank() over(partition by key order by value) as rr from src1;

-select *, rank() over(partition by key order by value) from src1;
\ No newline at end of file
+select *, rank() over(partition by key order by value) from src1;

Modified: hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/ctas_colname.q Mon Oct 27 22:59:12 2014
@@ -3,9 +3,11 @@
  -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table

  -- group by
+
+
  explain
-create table summary as select *, sum(key), count(value) from src;
-create table summary as select *, sum(key), count(value) from src;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
+create table summary as select *, key + 1, concat(value, value) from src limit 20;
  describe formatted summary;
  select * from summary;


Modified: hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/decimal_serde.q Mon Oct 27 22:59:12 2014
@@ -15,12 +15,15 @@ SELECT * FROM DECIMAL_TEXT ORDER BY key,
  CREATE TABLE DECIMAL_RC
  STORED AS RCFile AS
  SELECT * FROM DECIMAL_TEXT;
+describe formatted DECIMAL_RC;

  CREATE TABLE DECIMAL_LAZY_COL
  ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
  STORED AS RCFile AS
  SELECT * FROM DECIMAL_RC;

+describe formatted DECIMAL_LAZY_COL;
+
  CREATE TABLE DECIMAL_SEQUENCE
  ROW FORMAT DELIMITED
  FIELDS TERMINATED BY '\001'

Added: hive/trunk/ql/src/test/queries/clientpositive/insert0.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/insert0.q?rev=1634725&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/insert0.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/insert0.q Mon Oct 27 22:59:12 2014
@@ -0,0 +1,38 @@
+set hive.cbo.enable=true;
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
+
+CREATE TABLE insert_into1 (key int, value string);
+
+INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10;
+
+select * from insert_into1 order by key;
+
+INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10;
+
+select * from insert_into1 order by key;
+
+create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10;
+
+describe extended ctas_table;
+
+select * from ctas_table order by key;
+
+
+set hive.exec.dynamic.partition=true;
+SET hive.exec.dynamic.partition.mode=nonstrict;
+
+create table ctas_part (key int, value string) partitioned by (modkey bigint);
+
+insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10;
+
+select * from ctas_part order by key;
+
+
+
+DROP TABLE insert_into1;
+DROP TABLE ctas_table;
+DROP TABLE ctas_part;
\ No newline at end of file

Modified: hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out Mon Oct 27 22:59:12 2014
@@ -3,16 +3,20 @@ PREHOOK: query: -- SORT_QUERY_RESULTS
  -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table

  -- group by
+
+
  explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
  PREHOOK: type: CREATETABLE_AS_SELECT
  POSTHOOK: query: -- SORT_QUERY_RESULTS

  -- HIVE-4392, column aliases from expressionRR (GBY, etc.) are not valid name for table

  -- group by
+
+
  explain
-create table summary as select *, sum(key), count(value) from src
+create table summary as select *, key + 1, concat(value, value) from src limit 20
  POSTHOOK: type: CREATETABLE_AS_SELECT
  STAGE DEPENDENCIES:
    Stage-1 is a root stage
@@ -28,31 +32,27 @@ STAGE PLANS:
              alias: src
              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
              Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: key (type: string), value (type: string), (key + 1) (type: double), concat(value, value) (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
                Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: sum(key), count(value)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
                  Reduce Output Operator
                    sort order:
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: double), _col1 (type: bigint)
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _col3 (type: string)
        Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0), count(VALUE._col1)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: double), _col1 (type: bigint), _col0 (type: double), _col1 (type: bigint)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: double), VALUE._col3 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 20
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
              File Output Operator
                compressed: false
- Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
                table:
                    input format: org.apache.hadoop.mapred.TextInputFormat
                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -68,7 +68,7 @@ STAGE PLANS:
    Stage: Stage-3
        Create Table Operator:
          Create Table
- columns: _col0 double, _col1 bigint, _c1 double, _c2 bigint
+ columns: key string, value string, _c1 double, _c2 string
            input format: org.apache.hadoop.mapred.TextInputFormat
            output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
            serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
@@ -77,12 +77,12 @@ STAGE PLANS:
    Stage: Stage-2
      Stats-Aggr Operator

-PREHOOK: query: create table summary as select *, sum(key), count(value) from src
+PREHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
  PREHOOK: type: CREATETABLE_AS_SELECT
  PREHOOK: Input: default@src
  PREHOOK: Output: database:default
  PREHOOK: Output: default@summary
-POSTHOOK: query: create table summary as select *, sum(key), count(value) from src
+POSTHOOK: query: create table summary as select *, key + 1, concat(value, value) from src limit 20
  POSTHOOK: type: CREATETABLE_AS_SELECT
  POSTHOOK: Input: default@src
  POSTHOOK: Output: database:default
@@ -95,10 +95,10 @@ POSTHOOK: type: DESCTABLE
  POSTHOOK: Input: default@summary
  # col_name data_type comment

-_col0 double
-_col1 bigint
+key string
+value string
  _c1 double
-_c2 bigint
+_c2 string

  # Detailed Table Information
  Database: default
@@ -110,9 +110,9 @@ Table Type: MANAGED_TABLE
  Table Parameters:
   COLUMN_STATS_ACCURATE true
   numFiles 1
- numRows 1
- rawDataSize 25
- totalSize 26
+ numRows 20
+ rawDataSize 620
+ totalSize 640
  #### A masked pattern was here ####

  # Storage Information
@@ -133,7 +133,26 @@ POSTHOOK: query: select * from summary
  POSTHOOK: type: QUERY
  POSTHOOK: Input: default@summary
  #### A masked pattern was here ####
-130091.0 500 130091.0 500
+128 val_128 129.0 val_128val_128
+150 val_150 151.0 val_150val_150
+165 val_165 166.0 val_165val_165
+193 val_193 194.0 val_193val_193
+213 val_213 214.0 val_213val_213
+224 val_224 225.0 val_224val_224
+238 val_238 239.0 val_238val_238
+255 val_255 256.0 val_255val_255
+265 val_265 266.0 val_265val_265
+27 val_27 28.0 val_27val_27
+273 val_273 274.0 val_273val_273
+278 val_278 279.0 val_278val_278
+311 val_311 312.0 val_311val_311
+369 val_369 370.0 val_369val_369
+401 val_401 402.0 val_401val_401
+409 val_409 410.0 val_409val_409
+484 val_484 485.0 val_484val_484
+66 val_66 67.0 val_66val_66
+86 val_86 87.0 val_86val_86
+98 val_98 99.0 val_98val_98
  PREHOOK: query: -- window functions
  explain
  create table x4 as select *, rank() over(partition by key order by value) as rr from src1

Modified: hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out?rev=1634725&r1=1634724&r2=1634725&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/decimal_serde.q.out Mon Oct 27 22:59:12 2014
@@ -96,6 +96,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
  POSTHOOK: Input: default@decimal_text
  POSTHOOK: Output: database:default
  POSTHOOK: Output: default@DECIMAL_RC
+PREHOOK: query: describe formatted DECIMAL_RC
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_rc
+POSTHOOK: query: describe formatted DECIMAL_RC
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_rc
+# col_name data_type comment
+
+key decimal(10,0)
+value int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 1
+ numRows 38
+ rawDataSize 157
+ totalSize 278
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
  PREHOOK: query: CREATE TABLE DECIMAL_LAZY_COL
  ROW FORMAT SERDE "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"
  STORED AS RCFile AS
@@ -112,6 +148,42 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
  POSTHOOK: Input: default@decimal_rc
  POSTHOOK: Output: database:default
  POSTHOOK: Output: default@DECIMAL_LAZY_COL
+PREHOOK: query: describe formatted DECIMAL_LAZY_COL
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@decimal_lazy_col
+POSTHOOK: query: describe formatted DECIMAL_LAZY_COL
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@decimal_lazy_col
+# col_name data_type comment
+
+key decimal(10,0)
+value int
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 1
+ numRows 38
+ rawDataSize 157
+ totalSize 278
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
  PREHOOK: query: CREATE TABLE DECIMAL_SEQUENCE
  ROW FORMAT DELIMITED
  FIELDS TERMINATED BY '\001'

Added: hive/trunk/ql/src/test/results/clientpositive/insert0.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/insert0.q.out?rev=1634725&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/insert0.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/insert0.q.out Mon Oct 27 22:59:12 2014
@@ -0,0 +1,208 @@
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: CREATE TABLE insert_into1 (key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT OVERWRITE TABLE insert_into1 SELECT * from src ORDER BY key LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+PREHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: INSERT INTO TABLE insert_into1 SELECT * from src ORDER BY key DESC LIMIT 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@insert_into1
+POSTHOOK: Lineage: insert_into1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: insert_into1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from insert_into1 order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from insert_into1 order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@insert_into1
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+10 val_10
+90 val_90
+90 val_90
+92 val_92
+95 val_95
+95 val_95
+96 val_96
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+PREHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: create table ctas_table as SELECT key, count(value) as foo from src GROUP BY key LIMIT 10
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: describe extended ctas_table
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ctas_table
+POSTHOOK: query: describe extended ctas_table
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ctas_table
+key string
+foo bigint
+
+#### A masked pattern was here ####
+PREHOOK: query: select * from ctas_table order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_table order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_table
+#### A masked pattern was here ####
+0 3
+10 1
+100 2
+103 2
+104 2
+105 1
+11 1
+111 1
+113 2
+114 1
+PREHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: create table ctas_part (key int, value string) partitioned by (modkey bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas_part
+PREHOOK: query: insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: insert overwrite table ctas_part partition (modkey)
+select key, value, ceil(key / 100) from src where key is not null limit 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ctas_part@modkey=1
+POSTHOOK: Output: default@ctas_part@modkey=2
+POSTHOOK: Output: default@ctas_part@modkey=3
+POSTHOOK: Output: default@ctas_part@modkey=4
+POSTHOOK: Output: default@ctas_part@modkey=5
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=2).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=3).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=4).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ctas_part PARTITION(modkey=5).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from ctas_part order by key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ctas_part
+PREHOOK: Input: default@ctas_part@modkey=1
+PREHOOK: Input: default@ctas_part@modkey=2
+PREHOOK: Input: default@ctas_part@modkey=3
+PREHOOK: Input: default@ctas_part@modkey=4
+PREHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+POSTHOOK: query: select * from ctas_part order by key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Input: default@ctas_part@modkey=1
+POSTHOOK: Input: default@ctas_part@modkey=2
+POSTHOOK: Input: default@ctas_part@modkey=3
+POSTHOOK: Input: default@ctas_part@modkey=4
+POSTHOOK: Input: default@ctas_part@modkey=5
+#### A masked pattern was here ####
+27 val_27 1
+86 val_86 1
+98 val_98 1
+165 val_165 2
+238 val_238 3
+255 val_255 3
+278 val_278 3
+311 val_311 4
+409 val_409 5
+484 val_484 5
+PREHOOK: query: DROP TABLE insert_into1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@insert_into1
+PREHOOK: Output: default@insert_into1
+POSTHOOK: query: DROP TABLE insert_into1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@insert_into1
+POSTHOOK: Output: default@insert_into1
+PREHOOK: query: DROP TABLE ctas_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_table
+PREHOOK: Output: default@ctas_table
+POSTHOOK: query: DROP TABLE ctas_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_table
+POSTHOOK: Output: default@ctas_table
+PREHOOK: query: DROP TABLE ctas_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ctas_part
+PREHOOK: Output: default@ctas_part
+POSTHOOK: query: DROP TABLE ctas_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ctas_part
+POSTHOOK: Output: default@ctas_part

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 27, '14 at 11:00p
activeOct 27, '14 at 11:00p
posts1
users1
websitehive.apache.org

1 user in discussion

Sershe: 1 post

People

Translate

site design / logo © 2021 Grokbase