FAQ
Author: jdere
Date: Mon Feb 2 19:19:01 2015
New Revision: 1656544

URL: http://svn.apache.org/r1656544
Log:
HIVE-5472: support a simple scalar which returns the current timestamp (Jason Dere, reviewed by Thejas Nair)

Added:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
     hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q
     hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
Modified:
     hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
     hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Feb 2 19:19:01 2015
@@ -842,20 +842,22 @@ public class HiveConf extends Configurat

      // test mode in hive mode
      HIVETESTMODE("hive.test.mode", false,
- "Whether Hive is running in test mode. If yes, it turns on sampling and prefixes the output tablename."),
+ "Whether Hive is running in test mode. If yes, it turns on sampling and prefixes the output tablename.",
+ false),
      HIVETESTMODEPREFIX("hive.test.mode.prefix", "test_",
- "In test mode, specfies prefixes for the output table"),
+ "In test mode, specfies prefixes for the output table", false),
      HIVETESTMODESAMPLEFREQ("hive.test.mode.samplefreq", 32,
          "In test mode, specfies sampling frequency for table, which is not bucketed,\n" +
          "For example, the following query:\n" +
          " INSERT OVERWRITE TABLE dest SELECT col1 from src\n" +
          "would be converted to\n" +
          " INSERT OVERWRITE TABLE test_dest\n" +
- " SELECT col1 from src TABLESAMPLE (BUCKET 1 out of 32 on rand(1))"),
+ " SELECT col1 from src TABLESAMPLE (BUCKET 1 out of 32 on rand(1))", false),
      HIVETESTMODENOSAMPLE("hive.test.mode.nosamplelist", "",
- "In test mode, specifies comma separated table names which would not apply sampling"),
- HIVETESTMODEDUMMYSTATAGGR("hive.test.dummystats.aggregator", "", "internal variable for test"),
- HIVETESTMODEDUMMYSTATPUB("hive.test.dummystats.publisher", "", "internal variable for test"),
+ "In test mode, specifies comma separated table names which would not apply sampling", false),
+ HIVETESTMODEDUMMYSTATAGGR("hive.test.dummystats.aggregator", "", "internal variable for test", false),
+ HIVETESTMODEDUMMYSTATPUB("hive.test.dummystats.publisher", "", "internal variable for test", false),
+ HIVETESTCURRENTTIMESTAMP("hive.test.currenttimestamp", null, "current timestamp for test", false),

      HIVEMERGEMAPFILES("hive.merge.mapfiles", true,
          "Merge small files at the end of a map-only job"),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Feb 2 19:19:01 2015
@@ -381,6 +381,8 @@ public class Driver implements CommandPr
      String queryId = QueryPlan.makeQueryId();
      conf.setVar(HiveConf.ConfVars.HIVEQUERYID, queryId);

+ SessionState.get().setupQueryCurrentTimestamp();
+
      try {
        command = new VariableSubstitution().substitute(conf,command);
        ctx = new Context(conf);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java Mon Feb 2 19:19:01 2015
@@ -304,6 +304,8 @@ public final class FunctionRegistry {
      registerUDF("~", UDFOPBitNot.class, true);

      registerGenericUDF("current_database", UDFCurrentDB.class);
+ registerGenericUDF("current_date", GenericUDFCurrentDate.class);
+ registerGenericUDF("current_timestamp", GenericUDFCurrentTimestamp.class);

      registerGenericUDF("isnull", GenericUDFOPNull.class);
      registerGenericUDF("isnotnull", GenericUDFOPNotNull.class);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Mon Feb 2 19:19:01 2015
@@ -270,6 +270,8 @@ KW_UNBOUNDED: 'UNBOUNDED';
  KW_PRECEDING: 'PRECEDING';
  KW_FOLLOWING: 'FOLLOWING';
  KW_CURRENT: 'CURRENT';
+KW_CURRENT_DATE: 'CURRENT_DATE';
+KW_CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP';
  KW_LESS: 'LESS';
  KW_MORE: 'MORE';
  KW_OVER: 'OVER';

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Mon Feb 2 19:19:01 2015
@@ -173,11 +173,29 @@ function
                              -> ^(TOK_FUNCTIONDI functionName (selectExpression+)?)
      ;

+nonParenthesizedFunction
+@init { gParent.pushMsg("non-parenthesized function name", state); }
+@after { gParent.popMsg(state); }
+ :
+ nonParenthesizedFunctionName
+ -> ^(TOK_FUNCTION nonParenthesizedFunctionName)
+ ;
+
+nonParenthesizedFunctionName
+@init { gParent.pushMsg("non-parenthesized function name", state); }
+@after { gParent.popMsg(state); }
+ :
+ KW_CURRENT_DATE | KW_CURRENT_TIMESTAMP
+ ;
+
  functionName
  @init { gParent.pushMsg("function name", state); }
  @after { gParent.popMsg(state); }
      : // Keyword IF is also a function name
      KW_IF | KW_ARRAY | KW_MAP | KW_STRUCT | KW_UNIONTYPE | functionIdentifier
+ |
+ // This allows current_timestamp() to work as well as current_timestamp
+ nonParenthesizedFunctionName
      ;

  castExpression
@@ -273,6 +291,7 @@ atomExpression
castExpression
caseExpression
whenExpression
+ | nonParenthesizedFunction
(functionName LPAREN) => function
tableOrColumn
LPAREN! expression RPAREN!
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Mon Feb 2 19:19:01 2015
@@ -25,6 +25,7 @@ import java.io.InputStream;
  import java.io.PrintStream;
  import java.net.URI;
  import java.net.URLClassLoader;
+import java.sql.Timestamp;
  import java.util.*;

  import org.apache.commons.io.FileUtils;
@@ -252,6 +253,11 @@ public class SessionState {
    private final Set<String> preReloadableAuxJars = new HashSet<String>();

    /**
+ * CURRENT_TIMESTAMP value for query
+ */
+ private Timestamp queryCurrentTimestamp;
+
+ /**
     * Get the lineage state stored in this session.
     *
     * @return LineageState
@@ -1410,4 +1416,27 @@ public class SessionState {
      return Integer.toString(nextValueTempTableSuffix++);
    }

+ /**
+ * Initialize current timestamp, other necessary query initialization.
+ */
+ public void setupQueryCurrentTimestamp() {
+ queryCurrentTimestamp = new Timestamp(System.currentTimeMillis());
+
+ // Provide a facility to set current timestamp during tests
+ if (conf.getBoolVar(ConfVars.HIVE_IN_TEST)) {
+ String overrideTimestampString =
+ HiveConf.getVar(conf, HiveConf.ConfVars.HIVETESTCURRENTTIMESTAMP, null);
+ if (overrideTimestampString != null && overrideTimestampString.length() > 0) {
+ queryCurrentTimestamp = Timestamp.valueOf(overrideTimestampString);
+ }
+ }
+ }
+
+ /**
+ * Get query current timestamp
+ * @return
+ */
+ public Timestamp getQueryCurrentTimestamp() {
+ return queryCurrentTimestamp;
+ }
  }

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentDate.java Mon Feb 2 19:19:01 2015
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.sql.Date;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+// If there is a new UDFType to describe a function that is deterministic within a query
+// but changes value between queries, this function would fall into that category.
+@UDFType(deterministic = true)
+@Description(name = "current_date",
+ value = "_FUNC_() - Returns the current date at the start of query evaluation."
+ + " All calls of current_date within the same query return the same value.")
+public class GenericUDFCurrentDate extends GenericUDF {
+
+ protected DateWritable currentDate;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ if (arguments.length != 0) {
+ throw new UDFArgumentLengthException(
+ "The function CURRENT_DATE does not take any arguments, but found "
+ + arguments.length);
+ }
+
+ if (currentDate == null) {
+ Date dateVal =
+ Date.valueOf(SessionState.get().getQueryCurrentTimestamp().toString().substring(0, 10));
+ currentDate = new DateWritable(dateVal);
+ }
+
+ return PrimitiveObjectInspectorFactory.writableDateObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ return currentDate;
+ }
+
+ public DateWritable getCurrentDate() {
+ return currentDate;
+ }
+
+ public void setCurrentDate(DateWritable currentDate) {
+ this.currentDate = currentDate;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "CURRENT_DATE()";
+ }
+
+
+ @Override
+ public void copyToNewInstance(Object newInstance) throws UDFArgumentException {
+ super.copyToNewInstance(newInstance);
+ // Need to preserve currentDate
+ GenericUDFCurrentDate other = (GenericUDFCurrentDate) newInstance;
+ if (this.currentDate != null) {
+ other.currentDate = new DateWritable(this.currentDate);
+ }
+ }
+}

Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java (added)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCurrentTimestamp.java Mon Feb 2 19:19:01 2015
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+// If there is a new UDFType to describe a function that is deterministic within a query
+// but changes value between queries, this function would fall into that category.
+@UDFType(deterministic = true)
+@Description(name = "current_timestamp",
+ value = "_FUNC_() - Returns the current timestamp at the start of query evaluation."
+ + " All calls of current_timestamp within the same query return the same value.")
+public class GenericUDFCurrentTimestamp extends GenericUDF {
+
+ protected TimestampWritable currentTimestamp;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ if (arguments.length != 0) {
+ throw new UDFArgumentLengthException(
+ "The function CURRENT_TIMESTAMP does not take any arguments, but found "
+ + arguments.length);
+ }
+
+ if (currentTimestamp == null) {
+ currentTimestamp = new TimestampWritable(SessionState.get().getQueryCurrentTimestamp());
+ }
+
+ return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ return currentTimestamp;
+ }
+
+ public TimestampWritable getCurrentTimestamp() {
+ return currentTimestamp;
+ }
+
+ public void setCurrentTimestamp(TimestampWritable currentTimestamp) {
+ this.currentTimestamp = currentTimestamp;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "CURRENT_TIMESTAMP()";
+ }
+
+ @Override
+ public void copyToNewInstance(Object newInstance) throws UDFArgumentException {
+ super.copyToNewInstance(newInstance);
+ // Need to preserve currentTimestamp
+ GenericUDFCurrentTimestamp other = (GenericUDFCurrentTimestamp) newInstance;
+ if (this.currentTimestamp != null) {
+ other.currentTimestamp = new TimestampWritable(this.currentTimestamp);
+ }
+ }
+}

Added: hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/current_date_timestamp.q Mon Feb 2 19:19:01 2015
@@ -0,0 +1,4 @@
+select current_timestamp = current_timestamp(), current_date = current_date() from src limit 5;
+
+set hive.test.currenttimestamp =2012-01-01 01:02:03;
+select current_date, current_timestamp from src limit 5;

Added: hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out?rev=1656544&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/current_date_timestamp.q.out Mon Feb 2 19:19:01 2015
@@ -0,0 +1,26 @@
+PREHOOK: query: select current_timestamp = current_timestamp(), current_date = current_date() from src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select current_timestamp = current_timestamp(), current_date = current_date() from src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+true true
+true true
+true true
+true true
+true true
+PREHOOK: query: select current_date, current_timestamp from src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select current_date, current_timestamp from src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03
+2012-01-01 2012-01-01 01:02:03

Modified: hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out?rev=1656544&r1=1656543&r2=1656544&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/show_functions.q.out Mon Feb 2 19:19:01 2015
@@ -52,6 +52,8 @@ covar_samp
  create_union
  cume_dist
  current_database
+current_date
+current_timestamp
  date_add
  date_sub
  datediff
@@ -228,6 +230,8 @@ covar_samp
  create_union
  cume_dist
  current_database
+current_date
+current_timestamp
  PREHOOK: query: SHOW FUNCTIONS '.*e$'
  PREHOOK: type: SHOWFUNCTIONS
  POSTHOOK: query: SHOW FUNCTIONS '.*e$'
@@ -236,6 +240,7 @@ assert_true
  case
  coalesce
  current_database
+current_date
  decode
  e
  encode
@@ -277,6 +282,7 @@ PREHOOK: query: SHOW FUNCTIONS '.*date.*
  PREHOOK: type: SHOWFUNCTIONS
  POSTHOOK: query: SHOW FUNCTIONS '.*date.*'
  POSTHOOK: type: SHOWFUNCTIONS
+current_date
  date_add
  date_sub
  datediff

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 2, '15 at 7:19p
activeFeb 2, '15 at 7:19p
posts1
users1
websitehive.apache.org

1 user in discussion

Jdere: 1 post

People

Translate

site design / logo © 2021 Grokbase