On MacOS (Mojave, same on High Sierra) with Python 3, after installing and using pyenv for some time, seemingly all operations that use a network connection return a UnicodeDecodeError.
So far I've tried using various versions of Python 3 (3.6.Xs, 3.7 tried), system provided python, uninstalling pyenv and reinstalling Python3 with brew, setting locale in ~/.bashrc, ~/.bash_profile, ~/.profile to en_US.UTF-8, running commands with prefixed PYTHONIOENCODING=utf-8 or LC_CTYPE=en_US.UTF-8.
The issue still presents when used with different Python 3 distributions (tried Anaconda, brew).
Returning to MacOs's python 2 fixed similar issue with python2 stack.
Examples (all issues present with different python3 distributions):
Using pip3 (pyenv 3.7)
pip3 install requests
Collecting requests
Exception:
Traceback (most recent call last):
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/basecommand.py", line 228, in main
status = self.run(options, args)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/commands/install.py", line 291, in run
resolver.resolve(requirement_set)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/resolve.py", line 103, in resolve
self._resolve_one(requirement_set, req)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/resolve.py", line 257, in _resolve_one
abstract_dist = self._get_abstract_dist_for(req_to_install)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/resolve.py", line 210, in _get_abstract_dist_for
self.require_hashes
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/operations/prepare.py", line 245, in prepare_linked_requirement
req.populate_link(finder, upgrade_allowed, require_hashes)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/req/req_install.py", line 307, in populate_link
self.link = finder.find_requirement(self, upgrade)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/index.py", line 484, in find_requirement
all_candidates = self.find_all_candidates(req.name)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/index.py", line 442, in find_all_candidates
for page in self._get_pages(url_locations, project_name):
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/index.py", line 587, in _get_pages
page = self._get_page(location)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/index.py", line 705, in _get_page
return HTMLPage.get_page(link, session=self.session)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/index.py", line 814, in get_page
"Cache-Control": "max-age=600",
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/sessions.py", line 521, in get
return self.request('GET', url, **kwargs)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/download.py", line 397, in request
return super(PipSession, self).request(method, url, *args, **kwargs)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/sessions.py", line 494, in request
prep = self.prepare_request(req)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/sessions.py", line 437, in prepare_request
hooks=merge_hooks(request.hooks, self.hooks),
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/models.py", line 309, in prepare
self.prepare_auth(auth, url)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/models.py", line 540, in prepare_auth
r = auth(self)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_internal/download.py", line 161, in __call__
netrc_auth = get_netrc_auth(req.url)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/site-packages/pip/_vendor/requests/utils.py", line 195, in get_netrc_auth
_netrc = netrc(netrc_path).authenticators(host)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/netrc.py", line 30, in __init__
self._parse(file, fp, default_netrc)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/netrc.py", line 39, in _parse
toplevel = tt = lexer.get_token()
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/shlex.py", line 105, in get_token
raw = self.read_token()
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/shlex.py", line 136, in read_token
nextchar = self.instream.read(1)
File "/Users/nikolaj/.pyenv/versions/3.7.1/lib/python3.7/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc5 in position 104: invalid continuation byte
Decoding responses to standard requests (on Anaconda)
Python 3.7.1 (default, Dec 14 2018, 13:28:58)
[Clang 4.0.1 (tags/RELEASE_401/final)] :: Anaconda, Inc. on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import requests
>>> requests.get("http://www.google.pl")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/anaconda3/lib/python3.7/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/anaconda3/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 519, in request
prep = self.prepare_request(req)
File "/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 449, in prepare_request
auth = get_netrc_auth(request.url)
File "/anaconda3/lib/python3.7/site-packages/requests/utils.py", line 203, in get_netrc_auth
_netrc = netrc(netrc_path).authenticators(host)
File "/anaconda3/lib/python3.7/netrc.py", line 30, in __init__
self._parse(file, fp, default_netrc)
File "/anaconda3/lib/python3.7/netrc.py", line 39, in _parse
toplevel = tt = lexer.get_token()
File "/anaconda3/lib/python3.7/shlex.py", line 105, in get_token
raw = self.read_token()
File "/anaconda3/lib/python3.7/shlex.py", line 136, in read_token
nextchar = self.instream.read(1)
File "/anaconda3/lib/python3.7/codecs.py", line 322, in decode
(result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc5 in position 104: invalid continuation byte
My locale configuration
locale
LANG="en_US.UTF-8"
LC_COLLATE="en_US.UTF-8"
LC_CTYPE="en_US.UTF-8"
LC_MESSAGES="en_US.UTF-8"
LC_MONETARY="en_US.UTF-8"
LC_NUMERIC="en_US.UTF-8"
LC_TIME="en_US.UTF-8"
LC_ALL="en_US.UTF-8"
You need to fix your ~/.netrc file to contain UTF-8-clean data. Currently it does not.
See the .netrc file documentation on what the file should contain.
The problem doesn't arise on Python 2, because there the file is not opened as Unicode text (Python 2 mostly operates in a binary data world).