FAQ
Author: jdere
Date: Thu Sep 18 22:56:12 2014
New Revision: 1626101

URL: http://svn.apache.org/r1626101
Log:
HIVE-8047: Lazy char/varchar are not using escape char defined in serde params (Jason Dere, reviewed by Thejas Nair)

Added:
     hive/trunk/data/files/data_with_escape.txt
     hive/trunk/ql/src/test/queries/clientpositive/escape3.q
     hive/trunk/ql/src/test/results/clientpositive/escape3.q.out
Modified:
     hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
     hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java

Added: hive/trunk/data/files/data_with_escape.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/data_with_escape.txt?rev=1626101&view=auto
==============================================================================
--- hive/trunk/data/files/data_with_escape.txt (added)
+++ hive/trunk/data/files/data_with_escape.txt Thu Sep 18 22:56:12 2014
@@ -0,0 +1,5 @@
+re\|ading|V\|A|100
+writ\|ing|MD|200
+w\|aiting|\|NC|300
+seein\|g|TN\||400
+runn\|ing|WV|500

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Thu Sep 18 22:56:12 2014
@@ -735,7 +735,7 @@ public class TestJdbcDriver2 {
      assertTrue(res.next());
      // skip the last (partitioning) column since it is always non-null
      for (int i = 1; i < meta.getColumnCount(); i++) {
- assertNull(res.getObject(i));
+ assertNull("Column " + i + " should be null", res.getObject(i));
      }
      // getXXX returns 0 for numeric types, false for boolean and null for other
      assertEquals(0, res.getInt(1));

Added: hive/trunk/ql/src/test/queries/clientpositive/escape3.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/escape3.q?rev=1626101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/escape3.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/escape3.q Thu Sep 18 22:56:12 2014
@@ -0,0 +1,48 @@
+-- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1;
+
+select * from escape3_1;
+
+-- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2;
+
+select * from escape3_2;
+
+-- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE;
+
+LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3;
+
+select * from escape3_3;
+
+DROP TABLE escape3_1;
+DROP TABLE escape3_2;
+DROP TABLE escape3_3;

Added: hive/trunk/ql/src/test/results/clientpositive/escape3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/escape3.q.out?rev=1626101&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/escape3.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/escape3.q.out Thu Sep 18 22:56:12 2014
@@ -0,0 +1,165 @@
+PREHOOK: query: -- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: -- with string
+CREATE TABLE escape3_1
+(
+GERUND STRING,
+ABBREV STRING,
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: select * from escape3_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_1
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: -- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: -- with varchar
+CREATE TABLE escape3_2
+(
+GERUND VARCHAR(10),
+ABBREV VARCHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: select * from escape3_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_2
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: -- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: -- with char
+CREATE TABLE escape3_3
+(
+GERUND CHAR(10),
+ABBREV CHAR(3),
+CODE SMALLINT
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '|' ESCAPED BY '\134'
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@escape3_3
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/data_with_escape.txt' INTO TABLE escape3_3
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@escape3_3
+PREHOOK: query: select * from escape3_3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@escape3_3
+#### A masked pattern was here ####
+POSTHOOK: query: select * from escape3_3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@escape3_3
+#### A masked pattern was here ####
+re|ading V|A 100
+writ|ing MD 200
+w|aiting |NC 300
+seein|g TN| 400
+runn|ing WV 500
+PREHOOK: query: DROP TABLE escape3_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_1
+PREHOOK: Output: default@escape3_1
+POSTHOOK: query: DROP TABLE escape3_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_1
+POSTHOOK: Output: default@escape3_1
+PREHOOK: query: DROP TABLE escape3_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_2
+PREHOOK: Output: default@escape3_2
+POSTHOOK: query: DROP TABLE escape3_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_2
+POSTHOOK: Output: default@escape3_2
+PREHOOK: query: DROP TABLE escape3_3
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@escape3_3
+PREHOOK: Output: default@escape3_3
+POSTHOOK: query: DROP TABLE escape3_3
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@escape3_3
+POSTHOOK: Output: default@escape3_3

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java Thu Sep 18 22:56:12 2014
@@ -55,14 +55,24 @@ public class LazyHiveChar extends

    @Override
    public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
        isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ } else {
+ String byteData = null;
+ try {
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+ }
      }
    }


Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java Thu Sep 18 22:56:12 2014
@@ -55,14 +55,24 @@ public class LazyHiveVarchar extends

    @Override
    public void init(ByteArrayRef bytes, int start, int length) {
- String byteData = null;
- try {
- byteData = Text.decode(bytes.getData(), start, length);
- data.set(byteData, maxLength);
+ if (oi.isEscaped()) {
+ Text textData = data.getTextValue();
+ // This is doing a lot of copying here, this could be improved by enforcing length
+ // at the same time as escaping rather than as separate steps.
+ LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+ oi.getEscapeChar(),textData);
+ data.set(textData.toString(), maxLength);
        isNull = false;
- } catch (CharacterCodingException e) {
- isNull = true;
- LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ } else {
+ try {
+ String byteData = null;
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(byteData, maxLength);
+ isNull = false;
+ } catch (CharacterCodingException e) {
+ isNull = true;
+ LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+ }
      }
    }


Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Thu Sep 18 22:56:12 2014
@@ -40,40 +40,7 @@ public class LazyString extends LazyPrim
      if (oi.isEscaped()) {
        byte escapeChar = oi.getEscapeChar();
        byte[] inputBytes = bytes.getData();
-
- // First calculate the length of the output string
- int outputLength = 0;
- for (int i = 0; i < length; i++) {
- if (inputBytes[start + i] != escapeChar) {
- outputLength++;
- } else {
- outputLength++;
- i++;
- }
- }
-
- // Copy the data over, so that the internal state of Text will be set to
- // the required outputLength.
- data.set(bytes.getData(), start, outputLength);
-
- // We need to copy the data byte by byte only in case the
- // "outputLength < length" (which means there is at least one escaped
- // byte.
- if (outputLength < length) {
- int k = 0;
- byte[] outputBytes = data.getBytes();
- for (int i = 0; i < length; i++) {
- byte b = inputBytes[start + i];
- if (b != escapeChar || i == length - 1) {
- outputBytes[k++] = b;
- } else {
- // get the next byte
- i++;
- outputBytes[k++] = inputBytes[start + i];
- }
- }
- assert (k == outputLength);
- }
+ LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data);
      } else {
        // if the data is not escaped, simply copy the data.
        data.set(bytes.getData(), start, length);

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Thu Sep 18 22:56:12 2014
@@ -412,6 +412,44 @@ public final class LazyUtils {
      }
    }

+ public static void copyAndEscapeStringDataToText(byte[] inputBytes, int start, int length,
+ byte escapeChar, Text data) {
+
+ // First calculate the length of the output string
+ int outputLength = 0;
+ for (int i = 0; i < length; i++) {
+ if (inputBytes[start + i] != escapeChar) {
+ outputLength++;
+ } else {
+ outputLength++;
+ i++;
+ }
+ }
+
+ // Copy the data over, so that the internal state of Text will be set to
+ // the required outputLength.
+ data.set(inputBytes, start, outputLength);
+
+ // We need to copy the data byte by byte only in case the
+ // "outputLength < length" (which means there is at least one escaped
+ // byte.
+ if (outputLength < length) {
+ int k = 0;
+ byte[] outputBytes = data.getBytes();
+ for (int i = 0; i < length; i++) {
+ byte b = inputBytes[start + i];
+ if (b != escapeChar || i == length - 1) {
+ outputBytes[k++] = b;
+ } else {
+ // get the next byte
+ i++;
+ outputBytes[k++] = inputBytes[start + i];
+ }
+ }
+ assert (k == outputLength);
+ }
+ }
+
    private LazyUtils() {
      // prevent instantiation
    }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -29,12 +29,21 @@ public class LazyHiveCharObjectInspector
      extends AbstractPrimitiveLazyObjectInspector<HiveCharWritable>
      implements HiveCharObjectInspector {

+ private boolean escaped;
+ private byte escapeChar;
+
    // no-arg ctor required for Kyro
    public LazyHiveCharObjectInspector() {
    }

    public LazyHiveCharObjectInspector(CharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveCharObjectInspector(CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
      super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
    }

    @Override
@@ -63,6 +72,14 @@ public class LazyHiveCharObjectInspector
      return ret;
    }

+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
    @Override
    public String toString() {
      return getTypeName();

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -29,12 +29,21 @@ public class LazyHiveVarcharObjectInspec
      extends AbstractPrimitiveLazyObjectInspector<HiveVarcharWritable>
      implements HiveVarcharObjectInspector {

+ private boolean escaped;
+ private byte escapeChar;
+
    // no-arg ctor required for Kyro
    public LazyHiveVarcharObjectInspector() {
    }

    public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo) {
+ this(typeInfo, false, (byte)0);
+ }
+
+ public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
      super(typeInfo);
+ this.escaped = escaped;
+ this.escapeChar = escapeChar;
    }

    @Override
@@ -63,6 +72,14 @@ public class LazyHiveVarcharObjectInspec
      return ret;
    }

+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
    @Override
    public String toString() {
      return getTypeName();

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Thu Sep 18 22:56:12 2014
@@ -78,8 +78,10 @@ public final class LazyPrimitiveObjectIn
      // prevent instantiation
    }

- private static HashMap<ArrayList<Object>, LazyStringObjectInspector> cachedLazyStringObjectInspector =
- new HashMap<ArrayList<Object>, LazyStringObjectInspector>();
+ // Lazy object inspectors for string/char/varchar will all be cached in the same map.
+ // Map key will be list of [typeInfo, isEscaped, escapeChar]
+ private static HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector> cachedLazyStringTypeOIs =
+ new HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector>();

    private static Map<PrimitiveTypeInfo, AbstractPrimitiveLazyObjectInspector<?>>
       cachedPrimitiveLazyObjectInspectors =
@@ -121,6 +123,10 @@ public final class LazyPrimitiveObjectIn
      switch(primitiveCategory) {
      case STRING:
        return getLazyStringObjectInspector(escaped, escapeChar);
+ case CHAR:
+ return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar);
+ case VARCHAR:
+ return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar);
      case BOOLEAN:
        return getLazyBooleanObjectInspector(extBoolean);
      default:
@@ -157,13 +163,44 @@ public final class LazyPrimitiveObjectIn

    public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) {
      ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(TypeInfoFactory.stringTypeInfo);
      signature.add(Boolean.valueOf(escaped));
      signature.add(Byte.valueOf(escapeChar));
- LazyStringObjectInspector result = cachedLazyStringObjectInspector
+ LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs
          .get(signature);
      if (result == null) {
        result = new LazyStringObjectInspector(escaped, escapeChar);
- cachedLazyStringObjectInspector.put(signature, result);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveCharObjectInspector getLazyHiveCharObjectInspector(
+ CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveCharObjectInspector result = (LazyHiveCharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveCharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
+ }
+ return result;
+ }
+
+ public static LazyHiveVarcharObjectInspector getLazyHiveVarcharObjectInspector(
+ VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+ ArrayList<Object> signature = new ArrayList<Object>();
+ signature.add(typeInfo);
+ signature.add(Boolean.valueOf(escaped));
+ signature.add(Byte.valueOf(escapeChar));
+ LazyHiveVarcharObjectInspector result = (LazyHiveVarcharObjectInspector) cachedLazyStringTypeOIs
+ .get(signature);
+ if (result == null) {
+ result = new LazyHiveVarcharObjectInspector(typeInfo, escaped, escapeChar);
+ cachedLazyStringTypeOIs.put(signature, result);
      }
      return result;
    }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -91,6 +91,9 @@ public class WritableHiveCharObjectInspe

    @Override
    public Object set(Object o, HiveChar value) {
+ if (value == null) {
+ return null;
+ }
      HiveCharWritable writable = (HiveCharWritable) o;
      writable.set(value, getMaxLength());
      return o;
@@ -98,6 +101,9 @@ public class WritableHiveCharObjectInspe

    @Override
    public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
      HiveCharWritable writable = (HiveCharWritable) o;
      writable.set(value, getMaxLength());
      return o;

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1626101&r1=1626100&r2=1626101&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Thu Sep 18 22:56:12 2014
@@ -96,6 +96,9 @@ implements SettableHiveVarcharObjectInsp

    @Override
    public Object set(Object o, HiveVarchar value) {
+ if (value == null) {
+ return null;
+ }
      HiveVarcharWritable writable = (HiveVarcharWritable)o;
      writable.set(value, getMaxLength());
      return o;
@@ -103,6 +106,9 @@ implements SettableHiveVarcharObjectInsp

    @Override
    public Object set(Object o, String value) {
+ if (value == null) {
+ return null;
+ }
      HiveVarcharWritable writable = (HiveVarcharWritable)o;
      writable.set(value, getMaxLength());
      return o;

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedSep 18, '14 at 10:56p
activeSep 18, '14 at 10:56p
posts1
users1
websitehive.apache.org

1 user in discussion

Jdere: 1 post

People

Translate

site design / logo © 2021 Grokbase