FAQ
Repository: hive
Updated Branches:
   refs/heads/master 28cf89266 -> 3bf2f679c


HIVE-10187: Avro backed tables don't handle cyclical or recursive records (Mark Wagner via cws)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3bf2f679
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3bf2f679
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3bf2f679

Branch: refs/heads/master
Commit: 3bf2f679cc7b1da7d0c09041c9751c6bcd7e0d7e
Parents: 28cf892
Author: Carl Steinbach <csteinba@linkedin.com>
Authored: Fri Feb 12 12:55:55 2016 -0800
Committer: Carl Steinbach <csteinba@linkedin.com>
Committed: Fri Feb 12 12:55:55 2016 -0800

----------------------------------------------------------------------
  pom.xml | 6 +++
  .../hive/serde2/avro/SchemaToTypeInfo.java | 4 +-
  .../hive/serde2/avro/TestAvroSerializer.java | 44 ----------------
  .../hive/serde2/avro/TestSchemaToTypeInfo.java | 54 ++++++++++++++++++++
  4 files changed, 63 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 15e3522..af2e3d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -258,6 +258,12 @@
          <groupId>com.google.code.tempus-fugit</groupId>
          <artifactId>tempus-fugit</artifactId>
          <version>${tempus-fugit.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ </exclusion>
+ </exclusions>
        </dependency>
        <dependency>
          <groupId>com.googlecode.javaewah</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
index e60168c..35d83bd 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/SchemaToTypeInfo.java
@@ -217,7 +217,9 @@ class SchemaToTypeInfo {
      if (seenSchemas == null) {
          seenSchemas = Collections.newSetFromMap(new IdentityHashMap<Schema, Boolean>());
      } else if (seenSchemas.contains(schema)) {
- return primitiveTypeToTypeInfo.get(Schema.Type.NULL);
+ throw new AvroSerdeException(
+ "Recursive schemas are not supported. Recursive schema was " + schema
+ .getFullName());
      }
      seenSchemas.add(schema);


http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
index 0b94dc5..45be2dd 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java
@@ -491,48 +491,4 @@ public class TestAvroSerializer {

      assertArrayEquals(fixed.bytes(), ((GenericData.Fixed) r.get("fixed1")).bytes());
    }
-
- @Test
- public void canSerializeCyclesInSchema() throws SerDeException, IOException {
- // Create parent-child avro-record and avro-schema
- AvroCycleParent parent = new AvroCycleParent();
- AvroCycleChild child = new AvroCycleChild();
- parent.setChild (child);
- Schema parentS = ReflectData.AllowNull.get().getSchema(AvroCycleParent.class);
- GenericData.Record parentRec = new GenericData.Record(parentS);
- Schema childS = ReflectData.AllowNull.get().getSchema(AvroCycleChild.class);
- GenericData.Record childRec = new GenericData.Record(childS);
- parentRec.put("child", childRec);
-
- // Initialize Avro SerDe
- AvroSerializer as = new AvroSerializer();
- AvroDeserializer ad = new AvroDeserializer();
- AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(parentS);
- ObjectInspector oi = aoig.getObjectInspector();
- List<String> columnNames = aoig.getColumnNames();
- List<TypeInfo> columnTypes = aoig.getColumnTypes();
-
- // Check serialization and deserialization
- AvroGenericRecordWritable agrw = Utils.serializeAndDeserializeRecord(parentRec);
- Object obj = ad.deserialize(columnNames, columnTypes, agrw, parentS);
-
- Writable result = as.serialize(obj, oi, columnNames, columnTypes, parentS);
- assertTrue(result instanceof AvroGenericRecordWritable);
- GenericRecord r2 = ((AvroGenericRecordWritable) result).getRecord();
- assertEquals(parentS, r2.getSchema());
- }
-
- private static class AvroCycleParent {
- AvroCycleChild child;
- public AvroCycleChild getChild () {return child;}
- public void setChild (AvroCycleChild child) {this.child = child;}
- }
-
- private static class AvroCycleChild {
- AvroCycleParent parent;
- AvroCycleChild next;
- Map <String, AvroCycleParent> map;
- public AvroCycleParent getParent () {return parent;}
- public void setParent (AvroCycleParent parent) {this.parent = parent;}
- }
  }

http://git-wip-us.apache.org/repos/asf/hive/blob/3bf2f679/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
new file mode 100644
index 0000000..af258c7
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/avro/TestSchemaToTypeInfo.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.avro;
+
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+
+public class TestSchemaToTypeInfo {
+
+ @Rule
+ public ExpectedException expect = ExpectedException.none();
+
+ @Test
+ public void testDisallowRecursiveSchema()
+ throws AvroSerdeException {
+
+ expect.expect(AvroSerdeException.class);
+ expect.expectMessage("Recursive schemas are not supported");
+
+ final String schemaString = "{\n"
+ + " \"type\" : \"record\",\n"
+ + " \"name\" : \"Cycle\",\n"
+ + " \"namespace\" : \"org.apache.hadoop.hive.serde2.avro\",\n"
+ + " \"fields\" : [ {\n"
+ + " \"name\" : \"child\",\n"
+ + " \"type\" : [ \"null\", \"Cycle\"],\n"
+ + " \"default\" : null\n"
+ + " } ]\n"
+ + "}";
+
+ List<TypeInfo> types = SchemaToTypeInfo.generateColumnTypes(new Schema.Parser().parse(schemaString));
+ }
+}
\ No newline at end of file

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 12, '16 at 8:57p
activeFeb 12, '16 at 8:57p
posts1
users1
websitehive.apache.org

1 user in discussion

Cws: 1 post

People

Translate

site design / logo © 2021 Grokbase