FAQ
Repository: hive
Updated Branches:
   refs/heads/master 0c1de975f -> 306a64024


HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua Xu, reviewed by Szehon Ho)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/306a6402
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/306a6402
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/306a6402

Branch: refs/heads/master
Commit: 306a640241bf22b97ed546c2a17a7e90e5bd73d6
Parents: 0c1de97
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Mon Nov 23 14:41:58 2015 -0500
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Nov 23 15:18:42 2015 -0500

----------------------------------------------------------------------
  .../hive/serde2/lazy/LazySerDeParameters.java | 6 ++-
  .../hive/serde2/lazy/LazySimpleSerDe.java | 38 +++++++++----
  .../hive/serde2/lazy/TestLazySimpleSerDe.java | 56 +++++++++++++++++---
  3 files changed, 82 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
index 54f6b2b..11af860 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
@@ -20,14 +20,14 @@ package org.apache.hadoop.hive.serde2.lazy;

  import java.util.ArrayList;
  import java.util.Arrays;
-import java.util.HashMap;
  import java.util.List;
-import java.util.Map;
  import java.util.Properties;

  import org.slf4j.Logger;
  import org.slf4j.LoggerFactory;
  import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
  import org.apache.hadoop.hive.serde.serdeConstants;
  import org.apache.hadoop.hive.serde2.SerDeException;
@@ -42,6 +42,8 @@ import org.apache.hive.common.util.HiveStringUtils;
   * SerDeParameters.
   *
   */
+@Public
+@Stable
  public class LazySerDeParameters implements LazyObjectInspectorParameters {
    public static final Logger LOG = LoggerFactory.getLogger(LazySerDeParameters.class.getName());
    public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};

http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
index 0a2f44c..ac2d39f 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
@@ -19,16 +19,14 @@
  package org.apache.hadoop.hive.serde2.lazy;

  import java.io.IOException;
-import java.util.ArrayList;
  import java.util.Arrays;
-import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.Properties;

-import org.apache.commons.lang.ArrayUtils;
  import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
  import org.apache.hadoop.hive.serde.serdeConstants;
  import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe;
  import org.apache.hadoop.hive.serde2.ByteStream;
@@ -36,7 +34,6 @@ import org.apache.hadoop.hive.serde2.SerDeException;
  import org.apache.hadoop.hive.serde2.SerDeSpec;
  import org.apache.hadoop.hive.serde2.SerDeStats;
  import org.apache.hadoop.hive.serde2.SerDeUtils;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
  import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
  import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
  import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
@@ -48,14 +45,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
  import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
  import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
  import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
  import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
  import org.apache.hadoop.io.BinaryComparable;
  import org.apache.hadoop.io.Text;
  import org.apache.hadoop.io.Writable;
-import org.apache.hive.common.util.HiveStringUtils;
-

  /**
   * LazySimpleSerDe can be used to read the same data format as
@@ -67,6 +60,8 @@ import org.apache.hive.common.util.HiveStringUtils;
   * Also LazySimpleSerDe outputs typed columns instead of treating all columns as
   * String like MetadataTypedColumnsetSerDe.
   */
+@Public
+@Stable
  @SerDeSpec(schemaProps = {
      serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
      serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM,
@@ -413,4 +408,29 @@ public class LazySimpleSerDe extends AbstractEncodingAwareSerDe {
      Text text = (Text)blob;
      return SerDeUtils.transformTextToUTF8(text, this.charset);
    }
+
+ /**
+ * This method is deprecated and is only used for backward compatibility.
+ * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration, Properties, String)
+ */
+ @Deprecated
+ public static SerDeParameters initSerdeParams(Configuration job,
+ Properties tbl,
+ String serdeName) throws SerDeException {
+ return new SerDeParameters(job, tbl, serdeName);
+ }
+
+ /**
+ * This class is deprecated and is only used for backward compatibility. Replace by
+ * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.
+ */
+ @Deprecated
+ public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters {
+
+ public SerDeParameters(Configuration job,
+ Properties tbl,
+ String serdeName) throws SerDeException {
+ super(job, tbl, serdeName);
+ }
+ }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/306a6402/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
index b11ce32..391edd4 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.lazy;
  import java.io.IOException;
  import java.util.List;
  import java.util.Properties;
+import java.util.Random;

  import junit.framework.TestCase;

@@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde.serdeConstants;
  import org.apache.hadoop.hive.serde2.ByteStream;
  import org.apache.hadoop.hive.serde2.SerDeException;
  import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
+import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
  import org.apache.hadoop.hive.serde2.io.ByteWritable;
  import org.apache.hadoop.hive.serde2.io.DoubleWritable;
  import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
  import org.apache.hadoop.hive.serde2.objectinspector.StructField;
  import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
  import org.apache.hadoop.io.BytesWritable;
  import org.apache.hadoop.io.IntWritable;
  import org.apache.hadoop.io.LongWritable;
  import org.apache.hadoop.io.Text;
+import org.junit.Test;

  /**
   * TestLazySimpleSerDe.
@@ -81,8 +89,6 @@ public class TestLazySimpleSerDe extends TestCase {
      }
    }

-
-
    /**
     * Test the LazySimpleSerDe class with LastColumnTakesRest option.
     */
@@ -167,20 +173,56 @@ public class TestLazySimpleSerDe extends TestCase {
        throw e;
      }
    }
-
- Object serializeAndDeserialize(List<Integer> o1, StructObjectInspector oi1,
+
+ /**
+ * Tests the deprecated usage of SerDeParameters.
+ *
+ */
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testSerDeParameters() throws SerDeException, IOException {
+ // Setup
+ LazySimpleSerDe serDe = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+
+ MyTestClass row = new MyTestClass();
+ ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
+ row.randomFill(new Random(1234), extraTypeInfo);
+
+ StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(MyTestClass.class,
+ ObjectInspectorOptions.JAVA);
+
+ String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
+ String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
+
+ Properties schema = new Properties();
+ schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
+ schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
+
+ SerDeUtils.initializeSerDe(serDe, conf, schema, null);
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName");
+
+ // Test
+ LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams);
+ assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get());
+ assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get());
+ }
+
+ private Object serializeAndDeserialize(Object row,
+ StructObjectInspector rowOI,
        LazySimpleSerDe serde,
        LazySerDeParameters serdeParams) throws IOException, SerDeException {
      ByteStream.Output serializeStream = new ByteStream.Output();
- LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams
          .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
          .isEscaped(), serdeParams.getEscapeChar(), serdeParams
          .getNeedsEscape());
+
      Text t = new Text(serializeStream.toByteArray());
      return serde.deserialize(t);
    }
-
-
+
    private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s,
        Object[] expectedFieldsData) throws SerDeException {
      // Get the row structure

Search Discussions

  • Aihuaxu at Nov 23, 2015 at 8:47 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 f1c1427e6 -> 8d4b600a4


    HIVE-12406: HIVE-9500 introduced incompatible change to LazySimpleSerDe public interface (Aihua Xu, reviewed by Szehon Ho)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d4b600a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d4b600a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d4b600a

    Branch: refs/heads/branch-1
    Commit: 8d4b600a4f42d54c42e38e54369a747d58222a8e
    Parents: f1c1427
    Author: Aihua Xu <aihuaxu@apache.org>
    Authored: Mon Nov 23 14:41:58 2015 -0500
    Committer: Aihua Xu <aihuaxu@apache.org>
    Committed: Mon Nov 23 15:38:47 2015 -0500

    ----------------------------------------------------------------------
      .../hive/serde2/lazy/LazySerDeParameters.java | 6 ++-
      .../hive/serde2/lazy/LazySimpleSerDe.java | 38 +++++++++----
      .../hive/serde2/lazy/TestLazySimpleSerDe.java | 56 +++++++++++++++++---
      3 files changed, 82 insertions(+), 18 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
    index 60d11a2..1e72c72 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
    @@ -20,12 +20,12 @@ package org.apache.hadoop.hive.serde2.lazy;

      import java.util.ArrayList;
      import java.util.Arrays;
    -import java.util.HashMap;
      import java.util.List;
    -import java.util.Map;
      import java.util.Properties;

      import org.apache.hadoop.conf.Configuration;
    +import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
    +import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hadoop.hive.serde.serdeConstants;
      import org.apache.hadoop.hive.serde2.SerDeException;
    @@ -40,6 +40,8 @@ import org.apache.hive.common.util.HiveStringUtils;
       * SerDeParameters.
       *
       */
    +@Public
    +@Stable
      public class LazySerDeParameters implements LazyObjectInspectorParameters {
        public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
        public static final String SERIALIZATION_EXTEND_NESTING_LEVELS

    http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    index c373047..5674d15 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    @@ -19,18 +19,16 @@
      package org.apache.hadoop.hive.serde2.lazy;

      import java.io.IOException;
    -import java.util.ArrayList;
      import java.util.Arrays;
    -import java.util.HashMap;
      import java.util.List;
      import java.util.Map;
      import java.util.Properties;

    -import org.apache.commons.lang.ArrayUtils;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.conf.Configuration;
    -import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
    +import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public;
    +import org.apache.hadoop.hive.common.classification.InterfaceStability.Stable;
      import org.apache.hadoop.hive.serde.serdeConstants;
      import org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe;
      import org.apache.hadoop.hive.serde2.ByteStream;
    @@ -38,7 +36,6 @@ import org.apache.hadoop.hive.serde2.SerDeException;
      import org.apache.hadoop.hive.serde2.SerDeSpec;
      import org.apache.hadoop.hive.serde2.SerDeStats;
      import org.apache.hadoop.hive.serde2.SerDeUtils;
    -import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
      import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParametersImpl;
      import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
    @@ -50,14 +47,10 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
      import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
    -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
    -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
      import org.apache.hadoop.io.BinaryComparable;
      import org.apache.hadoop.io.Text;
      import org.apache.hadoop.io.Writable;
    -import org.apache.hive.common.util.HiveStringUtils;
    -

      /**
       * LazySimpleSerDe can be used to read the same data format as
    @@ -69,6 +62,8 @@ import org.apache.hive.common.util.HiveStringUtils;
       * Also LazySimpleSerDe outputs typed columns instead of treating all columns as
       * String like MetadataTypedColumnsetSerDe.
       */
    +@Public
    +@Stable
      @SerDeSpec(schemaProps = {
          serdeConstants.LIST_COLUMNS, serdeConstants.LIST_COLUMN_TYPES,
          serdeConstants.FIELD_DELIM, serdeConstants.COLLECTION_DELIM, serdeConstants.MAPKEY_DELIM,
    @@ -423,4 +418,29 @@ public class LazySimpleSerDe extends AbstractEncodingAwareSerDe {
          Text text = (Text)blob;
          return SerDeUtils.transformTextToUTF8(text, this.charset);
        }
    +
    + /**
    + * This method is deprecated and is only used for backward compatibility.
    + * Replaced by @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters#LazySerDeParameters(Configuration, Properties, String)
    + */
    + @Deprecated
    + public static SerDeParameters initSerdeParams(Configuration job,
    + Properties tbl,
    + String serdeName) throws SerDeException {
    + return new SerDeParameters(job, tbl, serdeName);
    + }
    +
    + /**
    + * This class is deprecated and is only used for backward compatibility. Replace by
    + * @see org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters.
    + */
    + @Deprecated
    + public static class SerDeParameters extends org.apache.hadoop.hive.serde2.lazy.LazySerDeParameters {
    +
    + public SerDeParameters(Configuration job,
    + Properties tbl,
    + String serdeName) throws SerDeException {
    + super(job, tbl, serdeName);
    + }
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/8d4b600a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
    index b11ce32..391edd4 100644
    --- a/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
    +++ b/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java
    @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.lazy;
      import java.io.IOException;
      import java.util.List;
      import java.util.Properties;
    +import java.util.Random;

      import junit.framework.TestCase;

    @@ -29,15 +30,22 @@ import org.apache.hadoop.hive.serde.serdeConstants;
      import org.apache.hadoop.hive.serde2.ByteStream;
      import org.apache.hadoop.hive.serde2.SerDeException;
      import org.apache.hadoop.hive.serde2.SerDeUtils;
    +import org.apache.hadoop.hive.serde2.binarysortable.MyTestClass;
    +import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
      import org.apache.hadoop.hive.serde2.io.ByteWritable;
      import org.apache.hadoop.hive.serde2.io.DoubleWritable;
      import org.apache.hadoop.hive.serde2.io.ShortWritable;
    +import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
    +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
    +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
      import org.apache.hadoop.hive.serde2.objectinspector.StructField;
      import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
    +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
      import org.apache.hadoop.io.BytesWritable;
      import org.apache.hadoop.io.IntWritable;
      import org.apache.hadoop.io.LongWritable;
      import org.apache.hadoop.io.Text;
    +import org.junit.Test;

      /**
       * TestLazySimpleSerDe.
    @@ -81,8 +89,6 @@ public class TestLazySimpleSerDe extends TestCase {
          }
        }

    -
    -
        /**
         * Test the LazySimpleSerDe class with LastColumnTakesRest option.
         */
    @@ -167,20 +173,56 @@ public class TestLazySimpleSerDe extends TestCase {
            throw e;
          }
        }
    -
    - Object serializeAndDeserialize(List<Integer> o1, StructObjectInspector oi1,
    +
    + /**
    + * Tests the deprecated usage of SerDeParameters.
    + *
    + */
    + @Test
    + @SuppressWarnings("deprecation")
    + public void testSerDeParameters() throws SerDeException, IOException {
    + // Setup
    + LazySimpleSerDe serDe = new LazySimpleSerDe();
    + Configuration conf = new Configuration();
    +
    + MyTestClass row = new MyTestClass();
    + ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
    + row.randomFill(new Random(1234), extraTypeInfo);
    +
    + StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory
    + .getReflectionObjectInspector(MyTestClass.class,
    + ObjectInspectorOptions.JAVA);
    +
    + String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    + String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    +
    + Properties schema = new Properties();
    + schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
    + schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
    +
    + SerDeUtils.initializeSerDe(serDe, conf, schema, null);
    + SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName");
    +
    + // Test
    + LazyStruct data = (LazyStruct)serializeAndDeserialize(row, rowOI, serDe, serdeParams);
    + assertEquals((boolean)row.myBool, ((LazyBoolean)data.getField(0)).getWritableObject().get());
    + assertEquals((int)row.myInt, ((LazyInteger)data.getField(3)).getWritableObject().get());
    + }
    +
    + private Object serializeAndDeserialize(Object row,
    + StructObjectInspector rowOI,
            LazySimpleSerDe serde,
            LazySerDeParameters serdeParams) throws IOException, SerDeException {
          ByteStream.Output serializeStream = new ByteStream.Output();
    - LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
    + LazySimpleSerDe.serialize(serializeStream, row, rowOI, serdeParams
              .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
              .isEscaped(), serdeParams.getEscapeChar(), serdeParams
              .getNeedsEscape());
    +
          Text t = new Text(serializeStream.toByteArray());
          return serde.deserialize(t);
        }
    -
    -
    +
        private void deserializeAndSerialize(LazySimpleSerDe serDe, Text t, String s,
            Object[] expectedFieldsData) throws SerDeException {
          // Get the row structure

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 23, '15 at 8:18p
activeNov 23, '15 at 8:47p
posts2
users1
websitehive.apache.org

1 user in discussion

Aihuaxu: 2 posts

People

Translate

site design / logo © 2021 Grokbase