FAQ
Author: gunther
Date: Wed Jul 9 20:58:34 2014
New Revision: 1609297

URL: http://svn.apache.org/r1609297
Log:
Merge latest trunk into branch. (Gunther Hagleitner)

Added:
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
       - copied unchanged from r1609292, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_authorize_create_tbl.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_authorize_create_tbl.q
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_column_stats.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_column_stats.q
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_create_like_partitions.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_create_like_partitions.q
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_index.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_index.q
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_partitions.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_partitions.q
     hive/branches/cbo/ql/src/test/queries/clientnegative/temp_table_rename.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientnegative/temp_table_rename.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_create_temp_table.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/authorization_create_temp_table.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/empty_dir_in_table.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/empty_dir_in_table.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/show_create_table_temp_table.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/show_create_table_temp_table.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_external.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_external.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_gb1.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_gb1.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_join1.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_join1.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_names.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_names.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_options1.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_options1.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_precedence.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_precedence.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_subquery1.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_subquery1.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/temp_table_windowing_expressions.q
       - copied unchanged from r1609292, hive/trunk/ql/src/test/queries/clientpositive/temp_table_windowing_expressions.q
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_authorize_create_tbl.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_authorize_create_tbl.q.out
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_column_stats.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_column_stats.q.out
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_create_like_partitions.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_create_like_partitions.q.out
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_index.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_index.q.out
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_partitions.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_partitions.q.out
     hive/branches/cbo/ql/src/test/results/clientnegative/temp_table_rename.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientnegative/temp_table_rename.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/authorization_create_temp_table.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/authorization_create_temp_table.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/empty_dir_in_table.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/empty_dir_in_table.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_temp_table.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/show_create_table_temp_table.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_external.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_external.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_gb1.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_gb1.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_join1.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_join1.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_names.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_names.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_options1.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_options1.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_precedence.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_precedence.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_subquery1.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_subquery1.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_windowing_expressions.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/temp_table_windowing_expressions.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/tez/temp_table.q.out
       - copied unchanged from r1609292, hive/trunk/ql/src/test/results/clientpositive/tez/temp_table.q.out
Modified:
     hive/branches/cbo/ (props changed)
     hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
     hive/branches/cbo/itests/qtest/testconfiguration.properties
     hive/branches/cbo/metastore/if/hive_metastore.thrift
     hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
     hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
     hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java
     hive/branches/cbo/metastore/src/gen/thrift/gen-php/metastore/Types.php
     hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
     hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
     hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
     hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Context.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java
     hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
     hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
     hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
     hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
     hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java
     hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_9.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/columnstats_partlvl.q
     hive/branches/cbo/ql/src/test/queries/clientpositive/stats19.q
     hive/branches/cbo/ql/src/test/results/clientpositive/authorization_9.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_alter.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
     hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_serde.q.out
     hive/branches/cbo/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java

Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
   Merged /hive/trunk:r1608997-1609292

Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniMr.java Wed Jul 9 20:58:34 2014
@@ -213,6 +213,37 @@ public class TestJdbcWithMiniMr {
      stmt.execute("DROP TABLE " + tableName);
    }

+ @Test
+ public void testTempTable() throws Exception {
+ // Create temp table with current connection
+ String tempTableName = "tmp1";
+ stmt.execute("CREATE TEMPORARY TABLE " + tempTableName + " (key string, value string)");
+ stmt.execute("load data local inpath '"
+ + dataFilePath.toString() + "' into table " + tempTableName);
+
+ String resultVal = "val_238";
+ String queryStr = "SELECT * FROM " + tempTableName +
+ " where value = '" + resultVal + "'";
+ verifyResult(queryStr, resultVal, 2);
+
+ // A second connection should not be able to see the table
+ Connection conn2 = DriverManager.getConnection(miniHS2.getJdbcURL(dbName),
+ System.getProperty("user.name"), "bar");
+ Statement stmt2 = conn2.createStatement();
+ stmt2.execute("USE " + dbName);
+ boolean gotException = false;
+ try {
+ ResultSet res;
+ res = stmt2.executeQuery(queryStr);
+ } catch (SQLException err) {
+ // This is expected to fail.
+ assertTrue("Expecting table not found error, instead got: " + err,
+ err.getMessage().contains("Table not found"));
+ gotException = true;
+ }
+ assertTrue("Exception while querying non-existing temp table", gotException);
+ }
+
    private void checkForNotExist(ResultSet res) throws Exception {
      int numRows = 0;
      while (res.next()) {

Modified: hive/branches/cbo/itests/qtest/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/qtest/testconfiguration.properties?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/itests/qtest/testconfiguration.properties (original)
+++ hive/branches/cbo/itests/qtest/testconfiguration.properties Wed Jul 9 20:58:34 2014
@@ -1,5 +1,5 @@
-minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q
+minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q
  minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q
  minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q
-minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,cbo_co
  rrectness.q
+minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t
  able.q,cbo_correctness.q
  beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,
  exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwr
  ite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q

Modified: hive/branches/cbo/metastore/if/hive_metastore.thrift
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/if/hive_metastore.thrift?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/if/hive_metastore.thrift (original)
+++ hive/branches/cbo/metastore/if/hive_metastore.thrift Wed Jul 9 20:58:34 2014
@@ -230,6 +230,7 @@ struct Table {
    11: string viewExpandedText, // expanded view text, null for non-view
    12: string tableType, // table type enum, e.g. EXTERNAL_TABLE
    13: optional PrincipalPrivilegeSet privileges,
+ 14: optional bool temporary=false
  }

  struct Partition {

Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.cpp Wed Jul 9 20:58:34 2014
@@ -2455,8 +2455,8 @@ void swap(StorageDescriptor &a, StorageD
    swap(a.__isset, b.__isset);
  }

-const char* Table::ascii_fingerprint = "68640B4B66B355CF317429AF70D2C260";
-const uint8_t Table::binary_fingerprint[16] = {0x68,0x64,0x0B,0x4B,0x66,0xB3,0x55,0xCF,0x31,0x74,0x29,0xAF,0x70,0xD2,0xC2,0x60};
+const char* Table::ascii_fingerprint = "29EFB2A5970EF572039E5D94CC78AA85";
+const uint8_t Table::binary_fingerprint[16] = {0x29,0xEF,0xB2,0xA5,0x97,0x0E,0xF5,0x72,0x03,0x9E,0x5D,0x94,0xCC,0x78,0xAA,0x85};

  uint32_t Table::read(::apache::thrift::protocol::TProtocol* iprot) {

@@ -2609,6 +2609,14 @@ uint32_t Table::read(::apache::thrift::p
            xfer += iprot->skip(ftype);
          }
          break;
+ case 14:
+ if (ftype == ::apache::thrift::protocol::T_BOOL) {
+ xfer += iprot->readBool(this->temporary);
+ this->__isset.temporary = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
        default:
          xfer += iprot->skip(ftype);
          break;
@@ -2695,6 +2703,11 @@ uint32_t Table::write(::apache::thrift::
      xfer += this->privileges.write(oprot);
      xfer += oprot->writeFieldEnd();
    }
+ if (this->__isset.temporary) {
+ xfer += oprot->writeFieldBegin("temporary", ::apache::thrift::protocol::T_BOOL, 14);
+ xfer += oprot->writeBool(this->temporary);
+ xfer += oprot->writeFieldEnd();
+ }
    xfer += oprot->writeFieldStop();
    xfer += oprot->writeStructEnd();
    return xfer;
@@ -2715,6 +2728,7 @@ void swap(Table &a, Table &b) {
    swap(a.viewExpandedText, b.viewExpandedText);
    swap(a.tableType, b.tableType);
    swap(a.privileges, b.privileges);
+ swap(a.temporary, b.temporary);
    swap(a.__isset, b.__isset);
  }


Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h Wed Jul 9 20:58:34 2014
@@ -1332,7 +1332,7 @@ class StorageDescriptor {
  void swap(StorageDescriptor &a, StorageDescriptor &b);

  typedef struct _Table__isset {
- _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false) {}
+ _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false), temporary(true) {}
    bool tableName;
    bool dbName;
    bool owner;
@@ -1346,15 +1346,16 @@ typedef struct _Table__isset {
    bool viewExpandedText;
    bool tableType;
    bool privileges;
+ bool temporary;
  } _Table__isset;

  class Table {
   public:

- static const char* ascii_fingerprint; // = "68640B4B66B355CF317429AF70D2C260";
- static const uint8_t binary_fingerprint[16]; // = {0x68,0x64,0x0B,0x4B,0x66,0xB3,0x55,0xCF,0x31,0x74,0x29,0xAF,0x70,0xD2,0xC2,0x60};
+ static const char* ascii_fingerprint; // = "29EFB2A5970EF572039E5D94CC78AA85";
+ static const uint8_t binary_fingerprint[16]; // = {0x29,0xEF,0xB2,0xA5,0x97,0x0E,0xF5,0x72,0x03,0x9E,0x5D,0x94,0xCC,0x78,0xAA,0x85};

- Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType() {
+ Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType(), temporary(false) {
    }

    virtual ~Table() throw() {}
@@ -1372,6 +1373,7 @@ class Table {
    std::string viewExpandedText;
    std::string tableType;
    PrincipalPrivilegeSet privileges;
+ bool temporary;

    _Table__isset __isset;

@@ -1428,6 +1430,11 @@ class Table {
      __isset.privileges = true;
    }

+ void __set_temporary(const bool val) {
+ temporary = val;
+ __isset.temporary = true;
+ }
+
    bool operator == (const Table & rhs) const
    {
      if (!(tableName == rhs.tableName))
@@ -1458,6 +1465,10 @@ class Table {
        return false;
      else if (__isset.privileges && !(privileges == rhs.privileges))
        return false;
+ if (__isset.temporary != rhs.__isset.temporary)
+ return false;
+ else if (__isset.temporary && !(temporary == rhs.temporary))
+ return false;
      return true;
    }
    bool operator != (const Table &rhs) const {

Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java Wed Jul 9 20:58:34 2014
@@ -47,6 +47,7 @@ public class Table implements org.apache
    private static final org.apache.thrift.protocol.TField VIEW_EXPANDED_TEXT_FIELD_DESC = new org.apache.thrift.protocol.TField("viewExpandedText", org.apache.thrift.protocol.TType.STRING, (short)11);
    private static final org.apache.thrift.protocol.TField TABLE_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("tableType", org.apache.thrift.protocol.TType.STRING, (short)12);
    private static final org.apache.thrift.protocol.TField PRIVILEGES_FIELD_DESC = new org.apache.thrift.protocol.TField("privileges", org.apache.thrift.protocol.TType.STRUCT, (short)13);
+ private static final org.apache.thrift.protocol.TField TEMPORARY_FIELD_DESC = new org.apache.thrift.protocol.TField("temporary", org.apache.thrift.protocol.TType.BOOL, (short)14);

    private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
    static {
@@ -67,6 +68,7 @@ public class Table implements org.apache
    private String viewExpandedText; // required
    private String tableType; // required
    private PrincipalPrivilegeSet privileges; // optional
+ private boolean temporary; // optional

    /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
    public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -82,7 +84,8 @@ public class Table implements org.apache
      VIEW_ORIGINAL_TEXT((short)10, "viewOriginalText"),
      VIEW_EXPANDED_TEXT((short)11, "viewExpandedText"),
      TABLE_TYPE((short)12, "tableType"),
- PRIVILEGES((short)13, "privileges");
+ PRIVILEGES((short)13, "privileges"),
+ TEMPORARY((short)14, "temporary");

      private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();

@@ -123,6 +126,8 @@ public class Table implements org.apache
            return TABLE_TYPE;
          case 13: // PRIVILEGES
            return PRIVILEGES;
+ case 14: // TEMPORARY
+ return TEMPORARY;
          default:
            return null;
        }
@@ -166,8 +171,9 @@ public class Table implements org.apache
    private static final int __CREATETIME_ISSET_ID = 0;
    private static final int __LASTACCESSTIME_ISSET_ID = 1;
    private static final int __RETENTION_ISSET_ID = 2;
+ private static final int __TEMPORARY_ISSET_ID = 3;
    private byte __isset_bitfield = 0;
- private _Fields optionals[] = {_Fields.PRIVILEGES};
+ private _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.TEMPORARY};
    public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
    static {
      Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -200,11 +206,15 @@ public class Table implements org.apache
          new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
      tmpMap.put(_Fields.PRIVILEGES, new org.apache.thrift.meta_data.FieldMetaData("privileges", org.apache.thrift.TFieldRequirementType.OPTIONAL,
          new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, PrincipalPrivilegeSet.class)));
+ tmpMap.put(_Fields.TEMPORARY, new org.apache.thrift.meta_data.FieldMetaData("temporary", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
      metaDataMap = Collections.unmodifiableMap(tmpMap);
      org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Table.class, metaDataMap);
    }

    public Table() {
+ this.temporary = false;
+
    }

    public Table(
@@ -293,6 +303,7 @@ public class Table implements org.apache
      if (other.isSetPrivileges()) {
        this.privileges = new PrincipalPrivilegeSet(other.privileges);
      }
+ this.temporary = other.temporary;
    }

    public Table deepCopy() {
@@ -317,6 +328,8 @@ public class Table implements org.apache
      this.viewExpandedText = null;
      this.tableType = null;
      this.privileges = null;
+ this.temporary = false;
+
    }

    public String getTableName() {
@@ -641,6 +654,28 @@ public class Table implements org.apache
      }
    }

+ public boolean isTemporary() {
+ return this.temporary;
+ }
+
+ public void setTemporary(boolean temporary) {
+ this.temporary = temporary;
+ setTemporaryIsSet(true);
+ }
+
+ public void unsetTemporary() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __TEMPORARY_ISSET_ID);
+ }
+
+ /** Returns true if field temporary is set (has been assigned a value) and false otherwise */
+ public boolean isSetTemporary() {
+ return EncodingUtils.testBit(__isset_bitfield, __TEMPORARY_ISSET_ID);
+ }
+
+ public void setTemporaryIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __TEMPORARY_ISSET_ID, value);
+ }
+
    public void setFieldValue(_Fields field, Object value) {
      switch (field) {
      case TABLE_NAME:
@@ -747,6 +782,14 @@ public class Table implements org.apache
        }
        break;

+ case TEMPORARY:
+ if (value == null) {
+ unsetTemporary();
+ } else {
+ setTemporary((Boolean)value);
+ }
+ break;
+
      }
    }

@@ -791,6 +834,9 @@ public class Table implements org.apache
      case PRIVILEGES:
        return getPrivileges();

+ case TEMPORARY:
+ return Boolean.valueOf(isTemporary());
+
      }
      throw new IllegalStateException();
    }
@@ -828,6 +874,8 @@ public class Table implements org.apache
        return isSetTableType();
      case PRIVILEGES:
        return isSetPrivileges();
+ case TEMPORARY:
+ return isSetTemporary();
      }
      throw new IllegalStateException();
    }
@@ -962,6 +1010,15 @@ public class Table implements org.apache
          return false;
      }

+ boolean this_present_temporary = true && this.isSetTemporary();
+ boolean that_present_temporary = true && that.isSetTemporary();
+ if (this_present_temporary || that_present_temporary) {
+ if (!(this_present_temporary && that_present_temporary))
+ return false;
+ if (this.temporary != that.temporary)
+ return false;
+ }
+
      return true;
    }

@@ -1034,6 +1091,11 @@ public class Table implements org.apache
      if (present_privileges)
        builder.append(privileges);

+ boolean present_temporary = true && (isSetTemporary());
+ builder.append(present_temporary);
+ if (present_temporary)
+ builder.append(temporary);
+
      return builder.toHashCode();
    }

@@ -1175,6 +1237,16 @@ public class Table implements org.apache
          return lastComparison;
        }
      }
+ lastComparison = Boolean.valueOf(isSetTemporary()).compareTo(typedOther.isSetTemporary());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetTemporary()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.temporary, typedOther.temporary);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
      return 0;
    }

@@ -1288,6 +1360,12 @@ public class Table implements org.apache
        }
        first = false;
      }
+ if (isSetTemporary()) {
+ if (!first) sb.append(", ");
+ sb.append("temporary:");
+ sb.append(this.temporary);
+ first = false;
+ }
      sb.append(")");
      return sb.toString();
    }
@@ -1468,6 +1546,14 @@ public class Table implements org.apache
                org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
              }
              break;
+ case 14: // TEMPORARY
+ if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
+ struct.temporary = iprot.readBool();
+ struct.setTemporaryIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
            default:
              org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
          }
@@ -1557,6 +1643,11 @@ public class Table implements org.apache
            oprot.writeFieldEnd();
          }
        }
+ if (struct.isSetTemporary()) {
+ oprot.writeFieldBegin(TEMPORARY_FIELD_DESC);
+ oprot.writeBool(struct.temporary);
+ oprot.writeFieldEnd();
+ }
        oprot.writeFieldStop();
        oprot.writeStructEnd();
      }
@@ -1614,7 +1705,10 @@ public class Table implements org.apache
        if (struct.isSetPrivileges()) {
          optionals.set(12);
        }
- oprot.writeBitSet(optionals, 13);
+ if (struct.isSetTemporary()) {
+ optionals.set(13);
+ }
+ oprot.writeBitSet(optionals, 14);
        if (struct.isSetTableName()) {
          oprot.writeString(struct.tableName);
        }
@@ -1667,12 +1761,15 @@ public class Table implements org.apache
        if (struct.isSetPrivileges()) {
          struct.privileges.write(oprot);
        }
+ if (struct.isSetTemporary()) {
+ oprot.writeBool(struct.temporary);
+ }
      }

      @Override
      public void read(org.apache.thrift.protocol.TProtocol prot, Table struct) throws org.apache.thrift.TException {
        TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(13);
+ BitSet incoming = iprot.readBitSet(14);
        if (incoming.get(0)) {
          struct.tableName = iprot.readString();
          struct.setTableNameIsSet(true);
@@ -1748,6 +1845,10 @@ public class Table implements org.apache
          struct.privileges.read(iprot);
          struct.setPrivilegesIsSet(true);
        }
+ if (incoming.get(13)) {
+ struct.temporary = iprot.readBool();
+ struct.setTemporaryIsSet(true);
+ }
      }
    }


Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-php/metastore/Types.php
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-php/metastore/Types.php?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-php/metastore/Types.php (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-php/metastore/Types.php Wed Jul 9 20:58:34 2014
@@ -3162,6 +3162,7 @@ class Table {
    public $viewExpandedText = null;
    public $tableType = null;
    public $privileges = null;
+ public $temporary = false;

    public function __construct($vals=null) {
      if (!isset(self::$_TSPEC)) {
@@ -3233,6 +3234,10 @@ class Table {
            'type' => TType::STRUCT,
            'class' => '\metastore\PrincipalPrivilegeSet',
            ),
+ 14 => array(
+ 'var' => 'temporary',
+ 'type' => TType::BOOL,
+ ),
          );
      }
      if (is_array($vals)) {
@@ -3275,6 +3280,9 @@ class Table {
        if (isset($vals['privileges'])) {
          $this->privileges = $vals['privileges'];
        }
+ if (isset($vals['temporary'])) {
+ $this->temporary = $vals['temporary'];
+ }
      }
    }

@@ -3414,6 +3422,13 @@ class Table {
              $xfer += $input->skip($ftype);
            }
            break;
+ case 14:
+ if ($ftype == TType::BOOL) {
+ $xfer += $input->readBool($this->temporary);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
          default:
            $xfer += $input->skip($ftype);
            break;
@@ -3523,6 +3538,11 @@ class Table {
        $xfer += $this->privileges->write($output);
        $xfer += $output->writeFieldEnd();
      }
+ if ($this->temporary !== null) {
+ $xfer += $output->writeFieldBegin('temporary', TType::BOOL, 14);
+ $xfer += $output->writeBool($this->temporary);
+ $xfer += $output->writeFieldEnd();
+ }
      $xfer += $output->writeFieldStop();
      $xfer += $output->writeStructEnd();
      return $xfer;

Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Wed Jul 9 20:58:34 2014
@@ -2153,6 +2153,7 @@ class Table:
     - viewExpandedText
     - tableType
     - privileges
+ - temporary
    """

    thrift_spec = (
@@ -2170,9 +2171,10 @@ class Table:
      (11, TType.STRING, 'viewExpandedText', None, None, ), # 11
      (12, TType.STRING, 'tableType', None, None, ), # 12
      (13, TType.STRUCT, 'privileges', (PrincipalPrivilegeSet, PrincipalPrivilegeSet.thrift_spec), None, ), # 13
+ (14, TType.BOOL, 'temporary', None, False, ), # 14
    )

- def __init__(self, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None,):
+ def __init__(self, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None, temporary=thrift_spec[14][4],):
      self.tableName = tableName
      self.dbName = dbName
      self.owner = owner
@@ -2186,6 +2188,7 @@ class Table:
      self.viewExpandedText = viewExpandedText
      self.tableType = tableType
      self.privileges = privileges
+ self.temporary = temporary

    def read(self, iprot):
      if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
@@ -2275,6 +2278,11 @@ class Table:
            self.privileges.read(iprot)
          else:
            iprot.skip(ftype)
+ elif fid == 14:
+ if ftype == TType.BOOL:
+ self.temporary = iprot.readBool();
+ else:
+ iprot.skip(ftype)
        else:
          iprot.skip(ftype)
        iprot.readFieldEnd()
@@ -2344,6 +2352,10 @@ class Table:
        oprot.writeFieldBegin('privileges', TType.STRUCT, 13)
        self.privileges.write(oprot)
        oprot.writeFieldEnd()
+ if self.temporary is not None:
+ oprot.writeFieldBegin('temporary', TType.BOOL, 14)
+ oprot.writeBool(self.temporary)
+ oprot.writeFieldEnd()
      oprot.writeFieldStop()
      oprot.writeStructEnd()


Modified: hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original)
+++ hive/branches/cbo/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Wed Jul 9 20:58:34 2014
@@ -531,6 +531,7 @@ class Table
    VIEWEXPANDEDTEXT = 11
    TABLETYPE = 12
    PRIVILEGES = 13
+ TEMPORARY = 14

    FIELDS = {
      TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'},
@@ -545,7 +546,8 @@ class Table
      VIEWORIGINALTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewOriginalText'},
      VIEWEXPANDEDTEXT => {:type => ::Thrift::Types::STRING, :name => 'viewExpandedText'},
      TABLETYPE => {:type => ::Thrift::Types::STRING, :name => 'tableType'},
- PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}
+ PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true},
+ TEMPORARY => {:type => ::Thrift::Types::BOOL, :name => 'temporary', :default => false, :optional => true}
    }

    def struct_fields; FIELDS; end

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Wed Jul 9 20:58:34 2014
@@ -139,7 +139,7 @@ public class HiveMetaStoreClient impleme
    private boolean isConnected = false;
    private URI metastoreUris[];
    private final HiveMetaHookLoader hookLoader;
- private final HiveConf conf;
+ protected final HiveConf conf;
    private String tokenStrForm;
    private final boolean localMetaStore;

@@ -147,7 +147,7 @@ public class HiveMetaStoreClient impleme
    private int retries = 5;
    private int retryDelaySeconds = 0;

- static final private Log LOG = LogFactory.getLog("hive.metastore");
+ static final protected Log LOG = LogFactory.getLog("hive.metastore");

    public HiveMetaStoreClient(HiveConf conf)
      throws MetaException {
@@ -555,7 +555,8 @@ public class HiveMetaStoreClient impleme
      }
      boolean success = false;
      try {
- client.create_table_with_environment_context(tbl, envContext);
+ // Subclasses can override this step (for example, for temporary tables)
+ create_table_with_environment_context(tbl, envContext);
        if (hook != null) {
          hook.commitCreateTable(tbl);
        }
@@ -617,7 +618,8 @@ public class HiveMetaStoreClient impleme
         List<String> tableList = getAllTables(name);
         for (String table : tableList) {
           try {
- dropTable(name, table, deleteData, false);
+ // Subclasses can override this step (for example, for temporary tables)
+ dropTable(name, table, deleteData, false);
           } catch (UnsupportedOperationException e) {
             // Ignore Index tables, those will be dropped with parent tables
           }
@@ -771,7 +773,7 @@ public class HiveMetaStoreClient impleme
      }
      boolean success = false;
      try {
- client.drop_table_with_environment_context(dbname, name, deleteData, envContext);
+ drop_table_with_environment_context(dbname, name, deleteData, envContext);
        if (hook != null) {
          hook.commitDropTable(tbl, deleteData);
        }
@@ -1342,7 +1344,7 @@ public class HiveMetaStoreClient impleme
      return copy;
    }

- private Table deepCopy(Table table) {
+ protected Table deepCopy(Table table) {
      Table copy = null;
      if (table != null) {
        copy = new Table(table);
@@ -1382,6 +1384,14 @@ public class HiveMetaStoreClient impleme
      return copy;
    }

+ protected PrincipalPrivilegeSet deepCopy(PrincipalPrivilegeSet pps) {
+ PrincipalPrivilegeSet copy = null;
+ if (pps != null) {
+ copy = new PrincipalPrivilegeSet(pps);
+ }
+ return copy;
+ }
+
    private List<Partition> deepCopyPartitions(List<Partition> partitions) {
      return deepCopyPartitions(partitions, null);
    }
@@ -1727,4 +1737,15 @@ public class HiveMetaStoreClient impleme
      return client.get_functions(dbName, pattern);
    }

+ protected void create_table_with_environment_context(Table tbl, EnvironmentContext envContext)
+ throws AlreadyExistsException, InvalidObjectException,
+ MetaException, NoSuchObjectException, TException {
+ client.create_table_with_environment_context(tbl, envContext);
+ }
+
+ protected void drop_table_with_environment_context(String dbname, String name,
+ boolean deleteData, EnvironmentContext envContext) throws MetaException, TException,
+ NoSuchObjectException, UnsupportedOperationException {
+ client.drop_table_with_environment_context(dbname, name, deleteData, envContext);
+ }
  }

Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Wed Jul 9 20:58:34 2014
@@ -102,7 +102,7 @@ public class Warehouse {
    /**
     * Helper functions to convert IOException to MetaException
     */
- public FileSystem getFs(Path f) throws MetaException {
+ public static FileSystem getFs(Path f, Configuration conf) throws MetaException {
      try {
        return f.getFileSystem(conf);
      } catch (IOException e) {
@@ -111,6 +111,10 @@ public class Warehouse {
      return null;
    }

+ public FileSystem getFs(Path f) throws MetaException {
+ return getFs(f, conf);
+ }
+
    public static void closeFs(FileSystem fs) throws MetaException {
      try {
        if (fs != null) {
@@ -135,12 +139,16 @@ public class Warehouse {
     * Path to be canonicalized
     * @return Path with canonical scheme and authority
     */
- public Path getDnsPath(Path path) throws MetaException {
- FileSystem fs = getFs(path);
+ public static Path getDnsPath(Path path, Configuration conf) throws MetaException {
+ FileSystem fs = getFs(path, conf);
      return (new Path(fs.getUri().getScheme(), fs.getUri().getAuthority(), path
          .toUri().getPath()));
    }

+ public Path getDnsPath(Path path) throws MetaException {
+ return getDnsPath(path, conf);
+ }
+
    /**
     * Resolve the configured warehouse root dir with respect to the configuration
     * This involves opening the FileSystem corresponding to the warehouse root
@@ -174,7 +182,6 @@ public class Warehouse {
      return new Path(getWhRoot(), dbName.toLowerCase() + DATABASE_WAREHOUSE_SUFFIX);
    }

-
    public Path getTablePath(Database db, String tableName)
        throws MetaException {
      return getDnsPath(new Path(getDatabasePath(db), tableName.toLowerCase()));

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/Context.java Wed Jul 9 20:58:34 2014
@@ -124,12 +124,9 @@ public class Context {

      // local & non-local tmp location is configurable. however it is the same across
      // all external file systems
- nonLocalScratchPath =
- new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
- executionId);
- localScratchDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR),
- executionId).toUri().getPath();
- scratchDirPermission= HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ nonLocalScratchPath = new Path(SessionState.getHDFSSessionPath(conf), executionId);
+ localScratchDir = new Path(SessionState.getLocalSessionPath(conf), executionId).toUri().getPath();
+ scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
    }



Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsTask.java Wed Jul 9 20:58:34 2014
@@ -287,7 +287,8 @@ public class ColumnStatsTask extends Tas
        Table tbl = db.getTable(dbName,tableName);
        List<FieldSchema> partColSchema = tbl.getPartCols();
        // Partition columns are appended at end, we only care about stats column
- for (int i = 0; i < fields.size() - partColSchema.size(); i++) {
+ int numOfStatCols = isTblLevel ? fields.size() : fields.size() - partColSchema.size();
+ for (int i = 0; i < numOfStatCols; i++) {
          // Get the field objectInspector, fieldName and the field object.
          ObjectInspector foi = fields.get(i).getFieldObjectInspector();
          Object f = (list == null ? null : list.get(i));

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Jul 9 20:58:34 2014
@@ -2193,6 +2193,7 @@ public class DDLTask extends Task<DDLWor
    private int showCreateTable(Hive db, ShowCreateTableDesc showCreateTbl) throws HiveException {
      // get the create table statement for the table and populate the output
      final String EXTERNAL = "external";
+ final String TEMPORARY = "temporary";
      final String LIST_COLUMNS = "columns";
      final String TBL_COMMENT = "tbl_comment";
      final String LIST_PARTITIONS = "partitions";
@@ -2224,7 +2225,7 @@ public class DDLTask extends Task<DDLWor
          return 0;
        }

- createTab_str.append("CREATE <" + EXTERNAL + "> TABLE `");
+ createTab_str.append("CREATE <" + TEMPORARY + "><" + EXTERNAL + ">TABLE `");
        createTab_str.append(tableName + "`(\n");
        createTab_str.append("<" + LIST_COLUMNS + ">)\n");
        createTab_str.append("<" + TBL_COMMENT + ">\n");
@@ -2239,11 +2240,17 @@ public class DDLTask extends Task<DDLWor
        createTab_str.append("<" + TBL_PROPERTIES + ">)\n");
        ST createTab_stmt = new ST(createTab_str.toString());

+ // For cases where the table is temporary
+ String tbl_temp = "";
+ if (tbl.isTemporary()) {
+ duplicateProps.add("TEMPORARY");
+ tbl_temp = "TEMPORARY ";
+ }
        // For cases where the table is external
        String tbl_external = "";
        if (tbl.getTableType() == TableType.EXTERNAL_TABLE) {
          duplicateProps.add("EXTERNAL");
- tbl_external = "EXTERNAL";
+ tbl_external = "EXTERNAL ";
        }

        // Columns
@@ -2399,6 +2406,7 @@ public class DDLTask extends Task<DDLWor
          tbl_properties += StringUtils.join(realProps, ", \n");
        }

+ createTab_stmt.add(TEMPORARY, tbl_temp);
        createTab_stmt.add(EXTERNAL, tbl_external);
        createTab_stmt.add(LIST_COLUMNS, tbl_columns);
        createTab_stmt.add(TBL_COMMENT, tbl_comment);
@@ -4246,6 +4254,8 @@ public class DDLTask extends Task<DDLWor
        tbl.setSkewedColValues(crtTbl.getSkewedColValues());
      }

+ tbl.getTTable().setTemporary(crtTbl.isTemporary());
+
      tbl.setStoredAsSubDirectories(crtTbl.isStoredAsSubDirectories());

      tbl.setInputFormatClass(crtTbl.getInputFormat());
@@ -4393,6 +4403,8 @@ public class DDLTask extends Task<DDLWor
          params.putAll(crtTbl.getTblProps());
        }

+ tbl.getTTable().setTemporary(crtTbl.isTemporary());
+
        if (crtTbl.isExternal()) {
          tbl.setProperty("EXTERNAL", "TRUE");
          tbl.setTableType(TableType.EXTERNAL_TABLE);

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Wed Jul 9 20:58:34 2014
@@ -681,6 +681,10 @@ public class Hive {
        if (baseTbl.getTableType() == TableType.VIRTUAL_VIEW.toString()) {
          throw new HiveException("tableName="+ tableName +" is a VIRTUAL VIEW. Index on VIRTUAL VIEW is not supported.");
        }
+ if (baseTbl.isTemporary()) {
+ throw new HiveException("tableName=" + tableName
+ + " is a TEMPORARY TABLE. Index on TEMPORARY TABLE is not supported.");
+ }

        if (indexTblName == null) {
          indexTblName = MetaStoreUtils.getIndexTableName(dbName, tableName, indexName);
@@ -2486,7 +2490,7 @@ private void constructOneLBLocationMap(F
          }
        };
      return RetryingMetaStoreClient.getProxy(conf, hookLoader,
- HiveMetaStoreClient.class.getName());
+ SessionHiveMetaStoreClient.class.getName());
    }

    /**

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Wed Jul 9 20:58:34 2014
@@ -996,4 +996,8 @@ public class Table implements Serializab
        throw new RuntimeException("Cannot get path ", e);
      }
    }
+
+ public boolean isTemporary() {
+ return tTable.isTemporary();
+ }
  };

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Wed Jul 9 20:58:34 2014
@@ -300,8 +300,11 @@ public class ColumnStatsSemanticAnalyzer
        rewrittenQueryBuilder.append(numBitVectors);
        rewrittenQueryBuilder.append(" )");
      }
- for (FieldSchema fs : tbl.getPartCols()) {
- rewrittenQueryBuilder.append(" , " + fs.getName());
+
+ if (isPartitionStats) {
+ for (FieldSchema fs : tbl.getPartCols()) {
+ rewrittenQueryBuilder.append(" , " + fs.getName());
+ }
      }
      rewrittenQueryBuilder.append(" from ");
      rewrittenQueryBuilder.append(tbl.getTableName());
@@ -355,6 +358,7 @@ public class ColumnStatsSemanticAnalyzer
        originalTree = tree;
        boolean isPartitionStats = isPartitionLevelStats(tree);
        Map<String,String> partSpec = null;
+ checkIfTemporaryTable();
        checkForPartitionColumns(colNames, Utilities.getColumnNamesFromFieldSchema(tbl.getPartitionKeys()));
        validateSpecifiedColumnNames(colNames);

@@ -402,6 +406,13 @@ public class ColumnStatsSemanticAnalyzer
      }
    }

+ private void checkIfTemporaryTable() throws SemanticException {
+ if (tbl.isTemporary()) {
+ throw new SemanticException(tbl.getTableName()
+ + " is a temporary table. Column statistics are not supported on temporary tables.");
+ }
+ }
+
    @Override
    public void analyze(ASTNode ast, Context origCtx) throws SemanticException {
      QB qb;

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Wed Jul 9 20:58:34 2014
@@ -804,7 +804,7 @@ databaseComment
  createTableStatement
  @init { pushMsg("create table statement", state); }
  @after { popMsg(state); }
- : KW_CREATE (ext=KW_EXTERNAL)? KW_TABLE ifNotExists? name=tableName
+ : KW_CREATE (temp=KW_TEMPORARY)? (ext=KW_EXTERNAL)? KW_TABLE ifNotExists? name=tableName
        ( like=KW_LIKE likeName=tableName
           tableLocation?
           tablePropertiesPrefixed?
@@ -819,7 +819,7 @@ createTableStatement
           tablePropertiesPrefixed?
           (KW_AS selectStatementWithCTE)?
        )
- -> ^(TOK_CREATETABLE $name $ext? ifNotExists?
+ -> ^(TOK_CREATETABLE $name $temp? $ext? ifNotExists?
           ^(TOK_LIKETABLE $likeName?)
           columnNameTypeList?
           tableComment?

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Wed Jul 9 20:58:34 2014
@@ -92,6 +92,7 @@ public class ImportSemanticAnalyzer exte
              table.getTableName(),
              false, // isExternal: set to false here, can be overwritten by the
                     // IMPORT stmt
+ table.isTemporary(),
              table.getSd().getCols(),
              table.getPartitionKeys(),
              table.getSd().getBucketCols(),

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Jul 9 20:58:34 2014
@@ -19,6 +19,7 @@
  package org.apache.hadoop.hive.ql.parse;

  import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
+import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DATABASE_WAREHOUSE_SUFFIX;

  import java.io.IOException;
  import java.io.Serializable;
@@ -32,6 +33,7 @@ import java.util.Iterator;
  import java.util.LinkedHashMap;
  import java.util.List;
  import java.util.Map;
+import java.util.UUID;
  import java.util.Map.Entry;
  import java.util.Set;
  import java.util.TreeSet;
@@ -10170,6 +10172,7 @@ public class SemanticAnalyzer extends Ba
      Map<String, String> tblProps = null;
      boolean ifNotExists = false;
      boolean isExt = false;
+ boolean isTemporary = false;
      ASTNode selectStmt = null;
      final int CREATE_TABLE = 0; // regular CREATE TABLE
      final int CTLT = 1; // CREATE TABLE LIKE ... (CTLT)
@@ -10206,6 +10209,9 @@ public class SemanticAnalyzer extends Ba
        case HiveParser.KW_EXTERNAL:
          isExt = true;
          break;
+ case HiveParser.KW_TEMPORARY:
+ isTemporary = true;
+ break;
        case HiveParser.TOK_LIKETABLE:
          if (child.getChildCount() > 0) {
            likeTableName = getUnescapedName((ASTNode) child.getChild(0));
@@ -10329,6 +10335,27 @@ public class SemanticAnalyzer extends Ba
      String dbName = qualified.length == 1 ? SessionState.get().getCurrentDatabase() : qualified[0];
      Database database = getDatabase(dbName);
      outputs.add(new WriteEntity(database, WriteEntity.WriteType.DDL_SHARED));
+
+ if (isTemporary) {
+ if (partCols.size() > 0) {
+ throw new SemanticException("Partition columns are not supported on temporary tables");
+ }
+
+ if (location == null) {
+ // for temporary tables we set the location to something in the session's scratch dir
+ // it has the same life cycle as the tmp table
+ try {
+ // Generate a unique ID for temp table path.
+ // This path will be fixed for the life of the temp table.
+ Path path = new Path(SessionState.getTempTableSpace(conf), UUID.randomUUID().toString());
+ path = Warehouse.getDnsPath(path, conf);
+ location = path.toString();
+ } catch (MetaException err) {
+ throw new SemanticException("Error while generating temp table path:", err);
+ }
+ }
+ }
+
      // Handle different types of CREATE TABLE command
      CreateTableDesc crtTblDesc = null;
      switch (command_type) {
@@ -10336,7 +10363,7 @@ public class SemanticAnalyzer extends Ba
      case CREATE_TABLE: // REGULAR CREATE TABLE DDL
        tblProps = addDefaultProperties(tblProps);

- crtTblDesc = new CreateTableDesc(tableName, isExt, cols, partCols,
+ crtTblDesc = new CreateTableDesc(tableName, isExt, isTemporary, cols, partCols,
            bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
            rowFormatParams.fieldEscape,
            rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim,
@@ -10358,7 +10385,14 @@ public class SemanticAnalyzer extends Ba
      case CTLT: // create table like <tbl_name>
        tblProps = addDefaultProperties(tblProps);

- CreateTableLikeDesc crtTblLikeDesc = new CreateTableLikeDesc(tableName, isExt,
+ if (isTemporary) {
+ Table likeTable = getTableWithQN(likeTableName, false);
+ if (likeTable != null && likeTable.getPartCols().size() > 0) {
+ throw new SemanticException("Partition columns are not supported on temporary tables "
+ + "and source table in CREATE TABLE LIKE is partitioned.");
+ }
+ }
+ CreateTableLikeDesc crtTblLikeDesc = new CreateTableLikeDesc(tableName, isExt, isTemporary,
            storageFormat.inputFormat, storageFormat.outputFormat, location,
            shared.serde, shared.serdeProps, tblProps, ifNotExists, likeTableName);
        SessionState.get().setCommandType(HiveOperation.CREATETABLE);
@@ -10380,7 +10414,7 @@ public class SemanticAnalyzer extends Ba

        tblProps = addDefaultProperties(tblProps);

- crtTblDesc = new CreateTableDesc(dbName, tableName, isExt, cols, partCols,
+ crtTblDesc = new CreateTableDesc(dbName, tableName, isExt, isTemporary, cols, partCols,
            bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
            rowFormatParams.fieldEscape,
            rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim,

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableDesc.java Wed Jul 9 20:58:34 2014
@@ -77,11 +77,12 @@ public class CreateTableDesc extends DDL
    List<String> skewedColNames;
    List<List<String>> skewedColValues;
    boolean isStoredAsSubDirectories = false;
+ boolean isTemporary = false;

    public CreateTableDesc() {
    }

- public CreateTableDesc(String databaseName, String tableName, boolean isExternal,
+ public CreateTableDesc(String databaseName, String tableName, boolean isExternal, boolean isTemporary,
        List<FieldSchema> cols, List<FieldSchema> partCols,
        List<String> bucketCols, List<Order> sortCols, int numBuckets,
        String fieldDelim, String fieldEscape, String collItemDelim,
@@ -92,7 +93,7 @@ public class CreateTableDesc extends DDL
        Map<String, String> tblProps,
        boolean ifNotExists, List<String> skewedColNames, List<List<String>> skewedColValues) {

- this(tableName, isExternal, cols, partCols,
+ this(tableName, isExternal, isTemporary, cols, partCols,
          bucketCols, sortCols, numBuckets, fieldDelim, fieldEscape,
          collItemDelim, mapKeyDelim, lineDelim, comment, inputFormat,
          outputFormat, location, serName, storageHandler, serdeProps,
@@ -101,7 +102,7 @@ public class CreateTableDesc extends DDL
      this.databaseName = databaseName;
    }

- public CreateTableDesc(String tableName, boolean isExternal,
+ public CreateTableDesc(String tableName, boolean isExternal, boolean isTemporary,
        List<FieldSchema> cols, List<FieldSchema> partCols,
        List<String> bucketCols, List<Order> sortCols, int numBuckets,
        String fieldDelim, String fieldEscape, String collItemDelim,
@@ -113,6 +114,7 @@ public class CreateTableDesc extends DDL
        boolean ifNotExists, List<String> skewedColNames, List<List<String>> skewedColValues) {
      this.tableName = tableName;
      this.isExternal = isExternal;
+ this.isTemporary = isTemporary;
      this.bucketCols = new ArrayList<String>(bucketCols);
      this.sortCols = new ArrayList<Order>(sortCols);
      this.collItemDelim = collItemDelim;
@@ -535,4 +537,19 @@ public class CreateTableDesc extends DDL
      this.nullFormat = nullFormat;
    }

+ /**
+ * @return the isTemporary
+ */
+ @Explain(displayName = "isTemporary", displayOnlyOnTrue = true)
+ public boolean isTemporary() {
+ return isTemporary;
+ }
+
+ /**
+ * @param isTemporary table is Temporary or not.
+ */
+ public void setTemporary(boolean isTemporary) {
+ this.isTemporary = isTemporary;
+ }
+
  }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateTableLikeDesc.java Wed Jul 9 20:58:34 2014
@@ -38,16 +38,18 @@ public class CreateTableLikeDesc extends
    Map<String, String> tblProps;
    boolean ifNotExists;
    String likeTableName;
+ boolean isTemporary = false;

    public CreateTableLikeDesc() {
    }

- public CreateTableLikeDesc(String tableName, boolean isExternal,
+ public CreateTableLikeDesc(String tableName, boolean isExternal, boolean isTemporary,
        String defaultInputFormat, String defaultOutputFormat, String location,
        String defaultSerName, Map<String, String> defaultSerdeProps, Map<String, String> tblProps,
        boolean ifNotExists, String likeTableName) {
      this.tableName = tableName;
      this.isExternal = isExternal;
+ this.isTemporary = isTemporary;
      this.defaultInputFormat=defaultInputFormat;
      this.defaultOutputFormat=defaultOutputFormat;
      this.defaultSerName=defaultSerName;
@@ -168,4 +170,20 @@ public class CreateTableLikeDesc extends
    public void setTblProps(Map<String, String> tblProps) {
      this.tblProps = tblProps;
    }
+
+ /**
+ * @return the isTemporary
+ */
+ @Explain(displayName = "isTemporary", displayOnlyOnTrue = true)
+ public boolean isTemporary() {
+ return isTemporary;
+ }
+
+ /**
+ * @param isTemporary table is Temporary or not.
+ */
+ public void setTemporary(boolean isTemporary) {
+ this.isTemporary = isTemporary;
+ }
+
  }

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Jul 9 20:58:34 2014
@@ -19,6 +19,8 @@
  package org.apache.hadoop.hive.ql.session;
  import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;

+import com.google.common.base.Preconditions;
+
  import java.io.File;
  import java.io.IOException;
  import java.io.InputStream;
@@ -41,6 +43,7 @@ import org.apache.hadoop.conf.Configurat
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.FileUtil;
  import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
  import org.apache.hadoop.hive.common.JavaUtils;
  import org.apache.hadoop.hive.conf.HiveConf;
  import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -55,6 +58,7 @@ import org.apache.hadoop.hive.ql.log.Per
  import org.apache.hadoop.hive.ql.metadata.Hive;
  import org.apache.hadoop.hive.ql.metadata.HiveException;
  import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.Table;
  import org.apache.hadoop.hive.ql.plan.HiveOperation;
  import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
  import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
@@ -75,6 +79,12 @@ import org.apache.hadoop.util.Reflection
  public class SessionState {
    private static final Log LOG = LogFactory.getLog(SessionState.class);

+ private static final String TMP_PREFIX = "_tmp_space.db";
+ private static final String LOCAL_SESSION_PATH_KEY = "_hive.local.session.path";
+ private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
+ private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
+ private Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
+
    protected ClassLoader parentLoader;

    /**
@@ -173,6 +183,24 @@ public class SessionState {
    private final String userName;

    /**
+ * scratch path to use for all non-local (ie. hdfs) file system tmp folders
+ * @return Path for Scratch path for the current session
+ */
+ private Path hdfsSessionPath;
+
+ /**
+ * sub dir of hdfs session path. used to keep tmp tables
+ * @return Path for temporary tables created by the current session
+ */
+ private Path hdfsTmpTableSpace;
+
+ /**
+ * scratch directory to use for local file system tmp folders
+ * @return Path for local scratch directory for current session
+ */
+ private Path localSessionPath;
+
+ /**
     * Get the lineage state stored in this session.
     *
     * @return LineageState
@@ -335,6 +363,7 @@ public class SessionState {
        Hive.get(new HiveConf(startSs.conf)).getMSC();
        ShimLoader.getHadoopShims().getUGIForConf(startSs.conf);
        FileSystem.get(startSs.conf);
+ startSs.createSessionPaths(startSs.conf);
      } catch (Exception e) {
        // catch-all due to some exec time dependencies on session state
        // that would cause ClassNoFoundException otherwise
@@ -359,6 +388,95 @@ public class SessionState {
      return startSs;
    }

+ public static Path getLocalSessionPath(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String localPathString = conf.get(LOCAL_SESSION_PATH_KEY);
+ Preconditions.checkNotNull(localPathString,
+ "Conf local session path expected to be non-null");
+ return new Path(localPathString);
+ }
+ Preconditions.checkNotNull(ss.localSessionPath,
+ "Local session path expected to be non-null");
+ return ss.localSessionPath;
+ }
+
+ public static Path getHDFSSessionPath(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String sessionPathString = conf.get(HDFS_SESSION_PATH_KEY);
+ Preconditions.checkNotNull(sessionPathString,
+ "Conf non-local session path expected to be non-null");
+ return new Path(sessionPathString);
+ }
+ Preconditions.checkNotNull(ss.hdfsSessionPath,
+ "Non-local session path expected to be non-null");
+ return ss.hdfsSessionPath;
+ }
+
+ public static Path getTempTableSpace(Configuration conf) {
+ SessionState ss = SessionState.get();
+ if (ss == null) {
+ String tempTablePathString = conf.get(TMP_TABLE_SPACE_KEY);
+ Preconditions.checkNotNull(tempTablePathString,
+ "Conf temp table path expected to be non-null");
+ return new Path(tempTablePathString);
+ }
+ return ss.getTempTableSpace();
+ }
+
+ public Path getTempTableSpace() {
+ Preconditions.checkNotNull(this.hdfsTmpTableSpace,
+ "Temp table path expected to be non-null");
+ return this.hdfsTmpTableSpace;
+ }
+
+ private void dropSessionPaths(Configuration conf) throws IOException {
+ if (hdfsSessionPath != null) {
+ hdfsSessionPath.getFileSystem(conf).delete(hdfsSessionPath, true);
+ }
+ if (localSessionPath != null) {
+ localSessionPath.getFileSystem(conf).delete(localSessionPath, true);
+ }
+ }
+
+ private void createSessionPaths(Configuration conf) throws IOException {
+
+ String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION);
+ String sessionId = getSessionId();
+
+ // local & non-local tmp location is configurable. however it is the same across
+ // all external file systems
+ hdfsSessionPath =
+ new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
+ sessionId);
+ createPath(conf, hdfsSessionPath, scratchDirPermission);
+ conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString());
+
+ localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR),
+ sessionId);
+ createPath(conf, localSessionPath, scratchDirPermission);
+ conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString());
+ hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX);
+ createPath(conf, hdfsTmpTableSpace, scratchDirPermission);
+ conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString());
+ }
+
+ private void createPath(Configuration conf, Path p, String perm) throws IOException {
+ FileSystem fs = p.getFileSystem(conf);
+ p = new Path(fs.makeQualified(p).toString());
+ FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(), 8));
+
+ if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) {
+ throw new IOException("Cannot create directory: "
+ + p.toString());
+ }
+
+ // best effort to clean up if we don't shut down properly
+ fs.deleteOnExit(p);
+ }
+
+
    /**
     * Setup authentication and authorization plugins for this session.
     */
@@ -923,6 +1041,8 @@ public class SessionState {
      } finally {
        tezSessionState = null;
      }
+
+ dropSessionPaths(conf);
    }

    public AuthorizationMode getAuthorizationMode(){
@@ -992,4 +1112,8 @@ public class SessionState {
      conf.set(CONFIG_AUTHZ_SETTINGS_APPLIED_MARKER, Boolean.TRUE.toString());

    }
+
+ public Map<String, Map<String, Table>> getTempTables() {
+ return tempTables;
+ }
  }

Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezTask.java Wed Jul 9 20:58:34 2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.plan.Re
  import org.apache.hadoop.hive.ql.plan.TezEdgeProperty;
  import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
  import org.apache.hadoop.hive.ql.plan.TezWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
  import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
  import org.apache.hadoop.mapred.JobConf;
  import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -156,6 +157,7 @@ public class TestTezTask {
      conf = new JobConf();
      appLr = mock(LocalResource.class);

+ SessionState.start(new HiveConf());
      session = mock(TezSession.class);
      sessionState = mock(TezSessionState.class);
      when(sessionState.getSession()).thenReturn(session);
@@ -166,6 +168,7 @@ public class TestTezTask {

    @After
    public void tearDown() throws Exception {
+ SessionState.get().close();
      utils = null;
      work = null;
      task = null;

Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java Wed Jul 9 20:58:34 2014
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.hooks.R
  import org.apache.hadoop.hive.ql.hooks.WriteEntity;
  import org.apache.hadoop.hive.ql.metadata.Partition;
  import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
  import org.apache.log4j.Level;
  import org.apache.log4j.LogManager;
  import org.junit.After;
@@ -51,6 +52,7 @@ public class TestDbTxnManager {

    public TestDbTxnManager() throws Exception {
      TxnDbUtil.setConfValues(conf);
+ SessionState.start(conf);
      ctx = new Context(conf);
      LogManager.getRootLogger().setLevel(Level.DEBUG);
      tearDown();

Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMacroSemanticAnalyzer.java Wed Jul 9 20:58:34 2014
@@ -42,10 +42,10 @@ public class TestMacroSemanticAnalyzer {
    @Before
    public void setup() throws Exception {
      conf = new HiveConf();
+ SessionState.start(conf);
      context = new Context(conf);
      parseDriver = new ParseDriver();
      analyzer = new MacroSemanticAnalyzer(conf);
- SessionState.start(conf);
    }

    private ASTNode parse(String command) throws Exception {

Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/parse/authorization/TestHiveAuthorizationTaskFactory.java Wed Jul 9 20:58:34 2014
@@ -69,10 +69,10 @@ public class TestHiveAuthorizationTaskFa
      db = Mockito.mock(Hive.class);
      table = new Table(DB, TABLE);
      partition = new Partition(table);
+ SessionState.start(conf);
      context = new Context(conf);
      parseDriver = new ParseDriver();
      analyzer = new DDLSemanticAnalyzer(conf, db);
- SessionState.start(conf);
      Mockito.when(db.getTable(DB, TABLE, false)).thenReturn(table);
      Mockito.when(db.getPartition(table, new HashMap<String, String>(), false))
        .thenReturn(partition);

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_9.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_9.q?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_9.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/authorization_9.q Wed Jul 9 20:58:34 2014
@@ -15,3 +15,11 @@ grant select on table dummy to user hive
  grant select (key, value) on table dummy to user hive_test_user2;

  show grant on all;
+
+revoke select on database default from user hive_test_user;
+revoke select on table dummy from user hive_test_user;
+revoke select (key, value) on table dummy from user hive_test_user;
+
+revoke select on database default from user hive_test_user2;
+revoke select on table dummy from user hive_test_user2;
+revoke select (key, value) on table dummy from user hive_test_user2;

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/columnstats_partlvl.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/columnstats_partlvl.q?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/columnstats_partlvl.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/columnstats_partlvl.q Wed Jul 9 20:58:34 2014
@@ -25,3 +25,9 @@ analyze table Employee_Part partition (e

  describe formatted Employee_Part.employeeID partition (employeeSalary=2000.0);
  describe formatted Employee_Part.employeeName partition (employeeSalary=2000.0);
+
+explain
+analyze table Employee_Part compute statistics for columns;
+analyze table Employee_Part compute statistics for columns;
+
+describe formatted Employee_Part.employeeID;

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/stats19.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/stats19.q?rev=1609297&r1=1609296&r2=1609297&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/stats19.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/stats19.q Wed Jul 9 20:58:34 2014
@@ -17,7 +17,7 @@ set hive.stats.key.prefix.max.length=0;
  -- The stats key should be hashed since the max length is too small
  insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;

-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;

  -- The stats key should not be hashed since the max length is large enough
  insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;
@@ -41,7 +41,7 @@ insert overwrite table stats_part partit

  desc formatted stats_part partition (ds='2010-04-08', hr = '13');

-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;

  -- The stats key should not be hashed since the max length is large enough
  insert overwrite table stats_part partition (ds='2010-04-08', hr = '13') select key, value from src;
@@ -66,7 +66,7 @@ set hive.stats.key.prefix.max.length=0;
  -- The stats key should be hashed since the max length is too small
  insert overwrite table stats_part partition (ds='2010-04-08', hr) select key, value, '13' from src;

-set hive.stats.key.prefix.max.length=200;
+set hive.stats.key.prefix.max.length=250;

  -- The stats key should not be hashed since the max length is large enough
  insert overwrite table stats_part partition (ds='2010-04-08', hr) select key, value, '13' from src;

Search Discussions

  • Gunther at Jul 9, 2014 at 8:59 pm
    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/authorization_9.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/authorization_9.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/authorization_9.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/authorization_9.q.out Wed Jul 9 20:58:34 2014
    @@ -72,3 +72,35 @@ default dummy key hive_test_user USER S
      default dummy key hive_test_user2 USER Select false -1 hive_test_user
      default dummy value hive_test_user USER Select false -1 hive_test_user
      default dummy value hive_test_user2 USER Select false -1 hive_test_user
    +PREHOOK: query: revoke select on database default from user hive_test_user
    +PREHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: query: revoke select on database default from user hive_test_user
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: query: revoke select on table dummy from user hive_test_user
    +PREHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: Output: default@dummy
    +POSTHOOK: query: revoke select on table dummy from user hive_test_user
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: Output: default@dummy
    +PREHOOK: query: revoke select (key, value) on table dummy from user hive_test_user
    +PREHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: Output: default@dummy
    +POSTHOOK: query: revoke select (key, value) on table dummy from user hive_test_user
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: Output: default@dummy
    +PREHOOK: query: revoke select on database default from user hive_test_user2
    +PREHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: query: revoke select on database default from user hive_test_user2
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: query: revoke select on table dummy from user hive_test_user2
    +PREHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: Output: default@dummy
    +POSTHOOK: query: revoke select on table dummy from user hive_test_user2
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: Output: default@dummy
    +PREHOOK: query: revoke select (key, value) on table dummy from user hive_test_user2
    +PREHOOK: type: REVOKE_PRIVILEGE
    +PREHOOK: Output: default@dummy
    +POSTHOOK: query: revoke select (key, value) on table dummy from user hive_test_user2
    +POSTHOOK: type: REVOKE_PRIVILEGE
    +POSTHOOK: Output: default@dummy

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/columnstats_partlvl.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/columnstats_partlvl.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    Files hive/branches/cbo/ql/src/test/results/clientpositive/columnstats_partlvl.q.out (original) and hive/branches/cbo/ql/src/test/results/clientpositive/columnstats_partlvl.q.out Wed Jul 9 20:58:34 2014 differ

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out Wed Jul 9 20:58:34 2014
    @@ -76,7 +76,7 @@ PREHOOK: Input: default@null_tab1
      POSTHOOK: query: SHOW CREATE TABLE null_tab1
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: default@null_tab1
    -CREATE TABLE `null_tab1`(
    +CREATE TABLE `null_tab1`(
        `a` string,
        `b` string)
      ROW FORMAT DELIMITED

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out Wed Jul 9 20:58:34 2014
    @@ -155,7 +155,7 @@ PREHOOK: Input: default@null_tab3
      POSTHOOK: query: SHOW CREATE TABLE null_tab3
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: default@null_tab3
    -CREATE TABLE `null_tab3`(
    +CREATE TABLE `null_tab3`(
        `a` string,
        `b` string)
      ROW FORMAT DELIMITED

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_alter.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_alter.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_alter.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_alter.q.out Wed Jul 9 20:58:34 2014
    @@ -51,7 +51,7 @@ PREHOOK: Input: default@tmp_showcrt1
      POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: default@tmp_showcrt1
    -CREATE TABLE `tmp_showcrt1`(
    +CREATE TABLE `tmp_showcrt1`(
        `key` smallint,
        `value` float)
      COMMENT 'temporary table'

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out Wed Jul 9 20:58:34 2014
    @@ -29,7 +29,7 @@ PREHOOK: Input: tmp_feng@tmp_showcrt
      POSTHOOK: query: SHOW CREATE TABLE tmp_feng.tmp_showcrt
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: tmp_feng@tmp_showcrt
    -CREATE TABLE `tmp_feng.tmp_showcrt`(
    +CREATE TABLE `tmp_feng.tmp_showcrt`(
        `key` string,
        `value` int)
      ROW FORMAT SERDE

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_delimited.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_delimited.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_delimited.q.out Wed Jul 9 20:58:34 2014
    @@ -23,7 +23,7 @@ PREHOOK: Input: default@tmp_showcrt1
      POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: default@tmp_showcrt1
    -CREATE TABLE `tmp_showcrt1`(
    +CREATE TABLE `tmp_showcrt1`(
        `key` int,
        `value` string,
        `newvalue` bigint)

    Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_serde.q.out
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_serde.q.out?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_serde.q.out (original)
    +++ hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_serde.q.out Wed Jul 9 20:58:34 2014
    @@ -25,7 +25,7 @@ PREHOOK: Input: default@tmp_showcrt1
      POSTHOOK: query: SHOW CREATE TABLE tmp_showcrt1
      POSTHOOK: type: SHOW_CREATETABLE
      POSTHOOK: Input: default@tmp_showcrt1
    -CREATE TABLE `tmp_showcrt1`(
    +CREATE TABLE `tmp_showcrt1`(
        `key` int,
        `value` string,
        `newvalue` bigint)

    Modified: hive/branches/cbo/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
    URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1609297&r1=1609296&r2=1609297&view=diff
    ==============================================================================
    --- hive/branches/cbo/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
    +++ hive/branches/cbo/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Wed Jul 9 20:58:34 2014
    @@ -26,6 +26,7 @@ import java.net.URI;
      import java.net.URISyntaxException;
      import java.security.PrivilegedExceptionAction;
      import java.util.ArrayList;
    +import java.util.Arrays;
      import java.util.Collections;
      import java.util.HashSet;
      import java.util.List;
    @@ -67,6 +68,8 @@ import org.apache.hadoop.tools.HadoopArc
      import org.apache.hadoop.util.Progressable;
      import org.apache.hadoop.util.ToolRunner;

    +import com.google.common.primitives.Longs;
    +
      /**
       * Base implemention for shims against secure Hadoop 0.20.3/0.23.
       */
    @@ -97,18 +100,12 @@ public abstract class HadoopShimsSecure
            _isShrinked = false;
          }

    - public InputSplitShim(CombineFileSplit old) throws IOException {
    - super(old.getJob(), old.getPaths(), old.getStartOffsets(),
    - old.getLengths(), dedup(old.getLocations()));
    + public InputSplitShim(JobConf conf, Path[] paths, long[] startOffsets,
    + long[] lengths, String[] locations) throws IOException {
    + super(conf, paths, startOffsets, lengths, dedup(locations));
            _isShrinked = false;
          }

    - private static String[] dedup(String[] locations) {
    - Set<String> dedup = new HashSet<String>();
    - Collections.addAll(dedup, locations);
    - return dedup.toArray(new String[dedup.size()]);
    - }
    -
          @Override
          public void shrinkSplit(long length) {
            _isShrinked = true;
    @@ -338,12 +335,22 @@ public abstract class HadoopShimsSecure

            InputSplit[] splits = (InputSplit[]) super.getSplits(job, numSplits);

    - InputSplitShim[] isplits = new InputSplitShim[splits.length];
    + ArrayList<InputSplitShim> inputSplitShims = new ArrayList<InputSplitShim>();
            for (int pos = 0; pos < splits.length; pos++) {
    - isplits[pos] = new InputSplitShim((CombineFileSplit)splits[pos]);
    + CombineFileSplit split = (CombineFileSplit) splits[pos];
    + Set<Integer> dirIndices = getDirIndices(split.getPaths(), job);
    + if (dirIndices.size() != split.getPaths().length) {
    + List<Path> prunedPaths = prune(dirIndices, Arrays.asList(split.getPaths()));
    + List<Long> prunedStartOffsets = prune(dirIndices, Arrays.asList(
    + ArrayUtils.toObject(split.getStartOffsets())));
    + List<Long> prunedLengths = prune(dirIndices, Arrays.asList(
    + ArrayUtils.toObject(split.getLengths())));
    + inputSplitShims.add(new InputSplitShim(job, prunedPaths.toArray(new Path[prunedPaths.size()]),
    + Longs.toArray(prunedStartOffsets),
    + Longs.toArray(prunedLengths), split.getLocations()));
    + }
            }
    -
    - return isplits;
    + return inputSplitShims.toArray(new InputSplitShim[inputSplitShims.size()]);
          }

          public InputSplitShim getInputSplitShim() throws IOException {
    @@ -623,4 +630,37 @@ public abstract class HadoopShimsSecure
          int retval = shell.run(command);
          LOG.debug("Return value is :" + retval);
        }
    +
    + /**
    + * CombineFileInputFormat sometimes returns directories as splits, need to prune them.
    + */
    + private static Set<Integer> getDirIndices(Path[] paths, JobConf conf) throws IOException {
    + Set<Integer> result = new HashSet<Integer>();
    + for (int i = 0; i < paths.length; i++) {
    + FileSystem fs = paths[i].getFileSystem(conf);
    + if (!fs.isFile(paths[i])) {
    + result.add(i);
    + }
    + }
    + return result;
    + }
    +
    + private static <K> List<K> prune(Set<Integer> indicesToPrune, List<K> elms) {
    + List<K> result = new ArrayList<K>();
    + int i = 0;
    + for (K elm : elms) {
    + if (indicesToPrune.contains(i)) {
    + continue;
    + }
    + result.add(elm);
    + i++;
    + }
    + return result;
    + }
    +
    + private static String[] dedup(String[] locations) throws IOException {
    + Set<String> dedup = new HashSet<String>();
    + Collections.addAll(dedup, locations);
    + return dedup.toArray(new String[dedup.size()]);
    + }
      }

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 9, '14 at 8:59p
activeJul 9, '14 at 8:59p
posts2
users1
websitehive.apache.org

1 user in discussion

Gunther: 2 posts

People

Translate

site design / logo © 2021 Grokbase