当我把数据插入一个列名有空格的Parquet格式的表中时，出现了下面的错误。
使用cloudera版本的hive客户端
创建表testcolumnames（ First Name （弦）作为Parquet地板储存；在testcolumnames中插入选择“john smith”；
有解决这个问题的方法吗？我们也从spark 2.3代码中得到了这个错误。
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.IllegalArgumentException: field ended by ';': expected ';' but got 'name' at line 1: optional binary first name
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:248)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:583)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:527)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:636)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815)
at org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:84)
at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815)
at org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:98)
at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:157)
at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:497)
at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170)
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.IllegalArgumentException: field ended by ';': expected ';' but got 'name' at line 1: optional binary first name
at parquet.schema.MessageTypeParser.check(MessageTypeParser.java:212)
at parquet.schema.MessageTypeParser.addPrimitiveType(MessageTypeParser.java:185)
at parquet.schema.MessageTypeParser.addType(MessageTypeParser.java:111)
at parquet.schema.MessageTypeParser.addGroupTypeFields(MessageTypeParser.java:99)
at parquet.schema.MessageTypeParser.parse(MessageTypeParser.java:92)
at parquet.schema.MessageTypeParser.parseMessageType(MessageTypeParser.java:82)
at org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.getSchema(DataWritableWriteSupport.java:43)
at org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.init(DataWritableWriteSupport.java:48)
at parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:310)
at parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:287)
at org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper.<init>(ParquetRecordWriterWrapper.java:69)
at org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat.getParquerRecordWriterWrapper(MapredParquetOutputFormat.java:134)
at org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat.getHiveRecordWriter(MapredParquetOutputFormat.java:123)
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:260)
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:245)
... 18 more
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.IllegalArgumentException: field ended by ';': expected ';' but got 'name' at line 1: optional binary first name
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:248)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketForFileIdx(FileSinkOperator.java:583)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:527)
at org.apache.hadoop.hive.ql.exec.FileSinkOperator.closeOp(FileSinkOperator.java:974)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:598)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:610)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:610)
at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:610)
at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.close(ExecMapper.java:199)
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:61)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1924)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)
Caused by: java.lang.IllegalArgumentException: field ended by ';': expected ';' but got 'name' at line 1: optional binary first name
at parquet.schema.MessageTypeParser.check(MessageTypeParser.java:212)
at parquet.schema.MessageTypeParser.addPrimitiveType(MessageTypeParser.java:185)
at parquet.schema.MessageTypeParser.addType(MessageTypeParser.java:111)
at parquet.schema.MessageTypeParser.addGroupTypeFields(MessageTypeParser.java:99)
at parquet.schema.MessageTypeParser.parse(MessageTypeParser.java:92)
at parquet.schema.MessageTypeParser.parseMessageType(MessageTypeParser.java:82)
at org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.getSchema(DataWritableWriteSupport.java:43)
at org.apache.hadoop.hive.ql.io.parquet.write.DataWritableWriteSupport.init(DataWritableWriteSupport.java:48)
at parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:310)
at parquet.hadoop.ParquetOutputFormat.getRecordWriter(ParquetOutputFormat.java:287)
at org.apache.hadoop.hive.ql.io.parquet.write.ParquetRecordWriterWrapper.<init>(ParquetRecordWriterWrapper.java:69)
at org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat.getParquerRecordWriterWrapper(MapredParquetOutputFormat.java:134)
at org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat.getHiveRecordWriter(MapredParquetOutputFormat.java:123)
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getRecordWriter(HiveFileFormatUtils.java:260)
at org.apache.hadoop.hive.ql.io.HiveFileFormatUtils.getHiveRecordWriter(HiveFileFormatUtils.java:245)
... 16 more
2条答案
按热度按时间zed5wv101#
请参考以下网址：
https://issues.apache.org/jira/browse/parquet-677
看来这个问题还没有解决。
hec6srdp2#
来自Hive医生https://cwiki.apache.org/confluence/display/hive/languagemanual+ddl
表名和列名不区分大小写，但serde和属性名区分大小写。
在hive0.12及更早版本中，表名和列名中只允许使用字母数字和下划线字符。
在Hive0.13及更高版本中，列名可以包含任何unicode字符（请参阅hive-6013），但是，点（.）和冒号（：）在查询时会产生错误，因此在Hive1.2.0中不允许它们（请参阅hive-10120）。在backticks（`）中指定的任何列名都按字面意义处理。在反记号字符串中，使用双反记号（``）表示反记号字符。backtick引号还支持对表和列标识符使用保留关键字。
要恢复到0.13.0之前的行为并将列名限制为字母数字和下划线字符，请将配置属性hive.support.quoted.identifiers设置为none。在此配置中，反勾选的名称被解释为正则表达式。有关详细信息，请参阅在列名中支持带引号的标识符。