关于 Python 上传文件到 hdfs 上的问题

2018-07-16 14:40:40 +08:00
 awli

代码: import pyhdfs client = pyhdfs.HdfsClient(':')

可以 list

client.listdir('/') ['apps', 'benchmarks', 'data', 'gj_data', 'hbase', 'system', 'test', 'tmp', 'user']

不能把文件放到本地

client.copy_from_local(dest='/tmp/test', localsrc='/Users/1.py', user_name='hdfs')


JSONDecodeError Traceback (most recent call last) /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyhdfs.py in _json(response) 789 try: --> 790 return response.json() 791 except simplejson.scanner.JSONDecodeError:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/requests/models.py in json(self, **kwargs) 895 pass --> 896 return complexjson.loads(self.text, **kwargs) 897

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/simplejson/init.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, use_decimal, **kw) 517 and not use_decimal and not kw): --> 518 return _default_decoder.decode(s) 519 if cls is None:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/simplejson/decoder.py in decode(self, s, _w, _PY3) 369 s = str(s, self.encoding) --> 370 obj, end = self.raw_decode(s) 371 end = _w(s, end).end()

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/simplejson/decoder.py in raw_decode(self, s, idx, _w, _PY3) 399 idx += 3 --> 400 return self.scan_once(s, idx=_w(s, idx).end())

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

HdfsException Traceback (most recent call last) <ipython-input-20-c3f47894efe1> in <module>() ----> 1 client.copy_from_local('/Users/lw/Desktop/1.py', '/tmp/test/')

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyhdfs.py in copy_from_local(self, localsrc, dest, **kwargs) 751 """ 752 with io.open(localsrc, 'rb') as f: --> 753 self.create(dest, f, **kwargs) 754 755 def copy_to_local(self, src, localdest, **kwargs):

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyhdfs.py in create(self, path, data, **kwargs) 425 data_response = self._requests_session.put( 426 metadata_response.headers['location'], data=data, **self._requests_kwargs) --> 427 _check_response(data_response, expected_status=httplib.CREATED) 428 assert not data_response.content 429

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyhdfs.py in _check_response(response, expected_status) 797 if response.status_code == expected_status: 798 return --> 799 remote_exception = _json(response)['RemoteException'] 800 exception_name = remote_exception['exception'] 801 python_name = 'Hdfs' + exception_name

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyhdfs.py in _json(response) 791 except simplejson.scanner.JSONDecodeError: 792 raise HdfsException( --> 793 "Expected JSON. Is WebHDFS enabled? Got {!r}".format(response.text)) 794 795

HdfsException: Expected JSON. Is WebHDFS enabled? Got '<html><head><title>Apache Tomcat/6.0.53 - Error report</title><style></style> </head><body>

HTTP Status 400 - Data upload requests must have content-type set to 'application/octet-stream'


type Status report

message <u>Data upload requests must have content-type set to 'application/octet-stream'</u>

description <u>The request sent by the client was syntactically incorrect.</u>


Apache Tomcat/6.0.53

</body></html>'

2643 次点击
所在节点    Python
0 条回复

这是一个专为移动设备优化的页面(即为了让你能够在 Google 搜索结果里秒开这个页面),如果你希望参与 V2EX 社区的讨论,你可以继续到 V2EX 上打开本讨论主题的完整版本。

https://www.v2ex.com/t/471324

V2EX 是创意工作者们的社区,是一个分享自己正在做的有趣事物、交流想法,可以遇见新朋友甚至新机会的地方。

V2EX is a community of developers, designers and creative people.

© 2021 V2EX