You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

912 lines
31 KiB

  1. from __future__ import absolute_import
  2. import cgi
  3. import email.utils
  4. import hashlib
  5. import getpass
  6. import json
  7. import logging
  8. import mimetypes
  9. import os
  10. import platform
  11. import re
  12. import shutil
  13. import sys
  14. import tempfile
  15. try:
  16. import ssl # noqa
  17. HAS_TLS = True
  18. except ImportError:
  19. HAS_TLS = False
  20. from pip._vendor.six.moves.urllib import parse as urllib_parse
  21. from pip._vendor.six.moves.urllib import request as urllib_request
  22. import pip
  23. from pip.exceptions import InstallationError, HashMismatch
  24. from pip.models import PyPI
  25. from pip.utils import (splitext, rmtree, format_size, display_path,
  26. backup_dir, ask_path_exists, unpack_file,
  27. call_subprocess, ARCHIVE_EXTENSIONS)
  28. from pip.utils.filesystem import check_path_owner
  29. from pip.utils.logging import indent_log
  30. from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner
  31. from pip.locations import write_delete_marker_file
  32. from pip.vcs import vcs
  33. from pip._vendor import requests, six
  34. from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
  35. from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
  36. from pip._vendor.requests.models import Response
  37. from pip._vendor.requests.structures import CaseInsensitiveDict
  38. from pip._vendor.requests.packages import urllib3
  39. from pip._vendor.cachecontrol import CacheControlAdapter
  40. from pip._vendor.cachecontrol.caches import FileCache
  41. from pip._vendor.lockfile import LockError
  42. from pip._vendor.six.moves import xmlrpc_client
  43. __all__ = ['get_file_content',
  44. 'is_url', 'url_to_path', 'path_to_url',
  45. 'is_archive_file', 'unpack_vcs_link',
  46. 'unpack_file_url', 'is_vcs_url', 'is_file_url',
  47. 'unpack_http_url', 'unpack_url']
  48. logger = logging.getLogger(__name__)
  49. def user_agent():
  50. """
  51. Return a string representing the user agent.
  52. """
  53. data = {
  54. "installer": {"name": "pip", "version": pip.__version__},
  55. "python": platform.python_version(),
  56. "implementation": {
  57. "name": platform.python_implementation(),
  58. },
  59. }
  60. if data["implementation"]["name"] == 'CPython':
  61. data["implementation"]["version"] = platform.python_version()
  62. elif data["implementation"]["name"] == 'PyPy':
  63. if sys.pypy_version_info.releaselevel == 'final':
  64. pypy_version_info = sys.pypy_version_info[:3]
  65. else:
  66. pypy_version_info = sys.pypy_version_info
  67. data["implementation"]["version"] = ".".join(
  68. [str(x) for x in pypy_version_info]
  69. )
  70. elif data["implementation"]["name"] == 'Jython':
  71. # Complete Guess
  72. data["implementation"]["version"] = platform.python_version()
  73. elif data["implementation"]["name"] == 'IronPython':
  74. # Complete Guess
  75. data["implementation"]["version"] = platform.python_version()
  76. if sys.platform.startswith("linux"):
  77. distro = dict(filter(
  78. lambda x: x[1],
  79. zip(["name", "version", "id"], platform.linux_distribution()),
  80. ))
  81. libc = dict(filter(
  82. lambda x: x[1],
  83. zip(["lib", "version"], platform.libc_ver()),
  84. ))
  85. if libc:
  86. distro["libc"] = libc
  87. if distro:
  88. data["distro"] = distro
  89. if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
  90. data["distro"] = {"name": "OS X", "version": platform.mac_ver()[0]}
  91. if platform.system():
  92. data.setdefault("system", {})["name"] = platform.system()
  93. if platform.release():
  94. data.setdefault("system", {})["release"] = platform.release()
  95. if platform.machine():
  96. data["cpu"] = platform.machine()
  97. return "{data[installer][name]}/{data[installer][version]} {json}".format(
  98. data=data,
  99. json=json.dumps(data, separators=(",", ":"), sort_keys=True),
  100. )
  101. class MultiDomainBasicAuth(AuthBase):
  102. def __init__(self, prompting=True):
  103. self.prompting = prompting
  104. self.passwords = {}
  105. def __call__(self, req):
  106. parsed = urllib_parse.urlparse(req.url)
  107. # Get the netloc without any embedded credentials
  108. netloc = parsed.netloc.rsplit("@", 1)[-1]
  109. # Set the url of the request to the url without any credentials
  110. req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
  111. # Use any stored credentials that we have for this netloc
  112. username, password = self.passwords.get(netloc, (None, None))
  113. # Extract credentials embedded in the url if we have none stored
  114. if username is None:
  115. username, password = self.parse_credentials(parsed.netloc)
  116. if username or password:
  117. # Store the username and password
  118. self.passwords[netloc] = (username, password)
  119. # Send the basic auth with this request
  120. req = HTTPBasicAuth(username or "", password or "")(req)
  121. # Attach a hook to handle 401 responses
  122. req.register_hook("response", self.handle_401)
  123. return req
  124. def handle_401(self, resp, **kwargs):
  125. # We only care about 401 responses, anything else we want to just
  126. # pass through the actual response
  127. if resp.status_code != 401:
  128. return resp
  129. # We are not able to prompt the user so simple return the response
  130. if not self.prompting:
  131. return resp
  132. parsed = urllib_parse.urlparse(resp.url)
  133. # Prompt the user for a new username and password
  134. username = six.moves.input("User for %s: " % parsed.netloc)
  135. password = getpass.getpass("Password: ")
  136. # Store the new username and password to use for future requests
  137. if username or password:
  138. self.passwords[parsed.netloc] = (username, password)
  139. # Consume content and release the original connection to allow our new
  140. # request to reuse the same one.
  141. resp.content
  142. resp.raw.release_conn()
  143. # Add our new username and password to the request
  144. req = HTTPBasicAuth(username or "", password or "")(resp.request)
  145. # Send our new request
  146. new_resp = resp.connection.send(req, **kwargs)
  147. new_resp.history.append(resp)
  148. return new_resp
  149. def parse_credentials(self, netloc):
  150. if "@" in netloc:
  151. userinfo = netloc.rsplit("@", 1)[0]
  152. if ":" in userinfo:
  153. return userinfo.split(":", 1)
  154. return userinfo, None
  155. return None, None
  156. class LocalFSAdapter(BaseAdapter):
  157. def send(self, request, stream=None, timeout=None, verify=None, cert=None,
  158. proxies=None):
  159. pathname = url_to_path(request.url)
  160. resp = Response()
  161. resp.status_code = 200
  162. resp.url = request.url
  163. try:
  164. stats = os.stat(pathname)
  165. except OSError as exc:
  166. resp.status_code = 404
  167. resp.raw = exc
  168. else:
  169. modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
  170. content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
  171. resp.headers = CaseInsensitiveDict({
  172. "Content-Type": content_type,
  173. "Content-Length": stats.st_size,
  174. "Last-Modified": modified,
  175. })
  176. resp.raw = open(pathname, "rb")
  177. resp.close = resp.raw.close
  178. return resp
  179. def close(self):
  180. pass
  181. class SafeFileCache(FileCache):
  182. """
  183. A file based cache which is safe to use even when the target directory may
  184. not be accessible or writable.
  185. """
  186. def __init__(self, *args, **kwargs):
  187. super(SafeFileCache, self).__init__(*args, **kwargs)
  188. # Check to ensure that the directory containing our cache directory
  189. # is owned by the user current executing pip. If it does not exist
  190. # we will check the parent directory until we find one that does exist.
  191. # If it is not owned by the user executing pip then we will disable
  192. # the cache and log a warning.
  193. if not check_path_owner(self.directory):
  194. logger.warning(
  195. "The directory '%s' or its parent directory is not owned by "
  196. "the current user and the cache has been disabled. Please "
  197. "check the permissions and owner of that directory. If "
  198. "executing pip with sudo, you may want sudo's -H flag.",
  199. self.directory,
  200. )
  201. # Set our directory to None to disable the Cache
  202. self.directory = None
  203. def get(self, *args, **kwargs):
  204. # If we don't have a directory, then the cache should be a no-op.
  205. if self.directory is None:
  206. return
  207. try:
  208. return super(SafeFileCache, self).get(*args, **kwargs)
  209. except (LockError, OSError, IOError):
  210. # We intentionally silence this error, if we can't access the cache
  211. # then we can just skip caching and process the request as if
  212. # caching wasn't enabled.
  213. pass
  214. def set(self, *args, **kwargs):
  215. # If we don't have a directory, then the cache should be a no-op.
  216. if self.directory is None:
  217. return
  218. try:
  219. return super(SafeFileCache, self).set(*args, **kwargs)
  220. except (LockError, OSError, IOError):
  221. # We intentionally silence this error, if we can't access the cache
  222. # then we can just skip caching and process the request as if
  223. # caching wasn't enabled.
  224. pass
  225. def delete(self, *args, **kwargs):
  226. # If we don't have a directory, then the cache should be a no-op.
  227. if self.directory is None:
  228. return
  229. try:
  230. return super(SafeFileCache, self).delete(*args, **kwargs)
  231. except (LockError, OSError, IOError):
  232. # We intentionally silence this error, if we can't access the cache
  233. # then we can just skip caching and process the request as if
  234. # caching wasn't enabled.
  235. pass
  236. class InsecureHTTPAdapter(HTTPAdapter):
  237. def cert_verify(self, conn, url, verify, cert):
  238. conn.cert_reqs = 'CERT_NONE'
  239. conn.ca_certs = None
  240. class PipSession(requests.Session):
  241. timeout = None
  242. def __init__(self, *args, **kwargs):
  243. retries = kwargs.pop("retries", 0)
  244. cache = kwargs.pop("cache", None)
  245. insecure_hosts = kwargs.pop("insecure_hosts", [])
  246. super(PipSession, self).__init__(*args, **kwargs)
  247. # Attach our User Agent to the request
  248. self.headers["User-Agent"] = user_agent()
  249. # Attach our Authentication handler to the session
  250. self.auth = MultiDomainBasicAuth()
  251. # Create our urllib3.Retry instance which will allow us to customize
  252. # how we handle retries.
  253. retries = urllib3.Retry(
  254. # Set the total number of retries that a particular request can
  255. # have.
  256. total=retries,
  257. # A 503 error from PyPI typically means that the Fastly -> Origin
  258. # connection got interupted in some way. A 503 error in general
  259. # is typically considered a transient error so we'll go ahead and
  260. # retry it.
  261. status_forcelist=[503],
  262. # Add a small amount of back off between failed requests in
  263. # order to prevent hammering the service.
  264. backoff_factor=0.25,
  265. )
  266. # We want to _only_ cache responses on securely fetched origins. We do
  267. # this because we can't validate the response of an insecurely fetched
  268. # origin, and we don't want someone to be able to poison the cache and
  269. # require manual evication from the cache to fix it.
  270. if cache:
  271. secure_adapter = CacheControlAdapter(
  272. cache=SafeFileCache(cache, use_dir_lock=True),
  273. max_retries=retries,
  274. )
  275. else:
  276. secure_adapter = HTTPAdapter(max_retries=retries)
  277. # Our Insecure HTTPAdapter disables HTTPS validation. It does not
  278. # support caching (see above) so we'll use it for all http:// URLs as
  279. # well as any https:// host that we've marked as ignoring TLS errors
  280. # for.
  281. insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
  282. self.mount("https://", secure_adapter)
  283. self.mount("http://", insecure_adapter)
  284. # Enable file:// urls
  285. self.mount("file://", LocalFSAdapter())
  286. # We want to use a non-validating adapter for any requests which are
  287. # deemed insecure.
  288. for host in insecure_hosts:
  289. self.mount("https://{0}/".format(host), insecure_adapter)
  290. def request(self, method, url, *args, **kwargs):
  291. # Allow setting a default timeout on a session
  292. kwargs.setdefault("timeout", self.timeout)
  293. # Dispatch the actual request
  294. return super(PipSession, self).request(method, url, *args, **kwargs)
  295. def get_file_content(url, comes_from=None, session=None):
  296. """Gets the content of a file; it may be a filename, file: URL, or
  297. http: URL. Returns (location, content). Content is unicode."""
  298. if session is None:
  299. raise TypeError(
  300. "get_file_content() missing 1 required keyword argument: 'session'"
  301. )
  302. match = _scheme_re.search(url)
  303. if match:
  304. scheme = match.group(1).lower()
  305. if (scheme == 'file' and comes_from and
  306. comes_from.startswith('http')):
  307. raise InstallationError(
  308. 'Requirements file %s references URL %s, which is local'
  309. % (comes_from, url))
  310. if scheme == 'file':
  311. path = url.split(':', 1)[1]
  312. path = path.replace('\\', '/')
  313. match = _url_slash_drive_re.match(path)
  314. if match:
  315. path = match.group(1) + ':' + path.split('|', 1)[1]
  316. path = urllib_parse.unquote(path)
  317. if path.startswith('/'):
  318. path = '/' + path.lstrip('/')
  319. url = path
  320. else:
  321. # FIXME: catch some errors
  322. resp = session.get(url)
  323. resp.raise_for_status()
  324. if six.PY3:
  325. return resp.url, resp.text
  326. else:
  327. return resp.url, resp.content
  328. try:
  329. with open(url) as f:
  330. content = f.read()
  331. except IOError as exc:
  332. raise InstallationError(
  333. 'Could not open requirements file: %s' % str(exc)
  334. )
  335. return url, content
  336. _scheme_re = re.compile(r'^(http|https|file):', re.I)
  337. _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
  338. def is_url(name):
  339. """Returns true if the name looks like a URL"""
  340. if ':' not in name:
  341. return False
  342. scheme = name.split(':', 1)[0].lower()
  343. return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
  344. def url_to_path(url):
  345. """
  346. Convert a file: URL to a path.
  347. """
  348. assert url.startswith('file:'), (
  349. "You can only turn file: urls into filenames (not %r)" % url)
  350. _, netloc, path, _, _ = urllib_parse.urlsplit(url)
  351. # if we have a UNC path, prepend UNC share notation
  352. if netloc:
  353. netloc = '\\\\' + netloc
  354. path = urllib_request.url2pathname(netloc + path)
  355. return path
  356. def path_to_url(path):
  357. """
  358. Convert a path to a file: URL. The path will be made absolute and have
  359. quoted path parts.
  360. """
  361. path = os.path.normpath(os.path.abspath(path))
  362. url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
  363. return url
  364. def is_archive_file(name):
  365. """Return True if `name` is a considered as an archive file."""
  366. ext = splitext(name)[1].lower()
  367. if ext in ARCHIVE_EXTENSIONS:
  368. return True
  369. return False
  370. def unpack_vcs_link(link, location):
  371. vcs_backend = _get_used_vcs_backend(link)
  372. vcs_backend.unpack(location)
  373. def _get_used_vcs_backend(link):
  374. for backend in vcs.backends:
  375. if link.scheme in backend.schemes:
  376. vcs_backend = backend(link.url)
  377. return vcs_backend
  378. def is_vcs_url(link):
  379. return bool(_get_used_vcs_backend(link))
  380. def is_file_url(link):
  381. return link.url.lower().startswith('file:')
  382. def _check_hash(download_hash, link):
  383. if download_hash.digest_size != hashlib.new(link.hash_name).digest_size:
  384. logger.critical(
  385. "Hash digest size of the package %d (%s) doesn't match the "
  386. "expected hash name %s!",
  387. download_hash.digest_size, link, link.hash_name,
  388. )
  389. raise HashMismatch('Hash name mismatch for package %s' % link)
  390. if download_hash.hexdigest() != link.hash:
  391. logger.critical(
  392. "Hash of the package %s (%s) doesn't match the expected hash %s!",
  393. link, download_hash.hexdigest(), link.hash,
  394. )
  395. raise HashMismatch(
  396. 'Bad %s hash for package %s' % (link.hash_name, link)
  397. )
  398. def _get_hash_from_file(target_file, link):
  399. try:
  400. download_hash = hashlib.new(link.hash_name)
  401. except (ValueError, TypeError):
  402. logger.warning(
  403. "Unsupported hash name %s for package %s", link.hash_name, link,
  404. )
  405. return None
  406. with open(target_file, 'rb') as fp:
  407. while True:
  408. chunk = fp.read(4096)
  409. if not chunk:
  410. break
  411. download_hash.update(chunk)
  412. return download_hash
  413. def _progress_indicator(iterable, *args, **kwargs):
  414. return iterable
  415. def _download_url(resp, link, content_file):
  416. download_hash = None
  417. if link.hash and link.hash_name:
  418. try:
  419. download_hash = hashlib.new(link.hash_name)
  420. except ValueError:
  421. logger.warning(
  422. "Unsupported hash name %s for package %s",
  423. link.hash_name, link,
  424. )
  425. try:
  426. total_length = int(resp.headers['content-length'])
  427. except (ValueError, KeyError, TypeError):
  428. total_length = 0
  429. cached_resp = getattr(resp, "from_cache", False)
  430. if logger.getEffectiveLevel() > logging.INFO:
  431. show_progress = False
  432. elif cached_resp:
  433. show_progress = False
  434. elif total_length > (40 * 1000):
  435. show_progress = True
  436. elif not total_length:
  437. show_progress = True
  438. else:
  439. show_progress = False
  440. show_url = link.show_url
  441. def resp_read(chunk_size):
  442. try:
  443. # Special case for urllib3.
  444. for chunk in resp.raw.stream(
  445. chunk_size,
  446. # We use decode_content=False here because we do
  447. # want urllib3 to mess with the raw bytes we get
  448. # from the server. If we decompress inside of
  449. # urllib3 then we cannot verify the checksum
  450. # because the checksum will be of the compressed
  451. # file. This breakage will only occur if the
  452. # server adds a Content-Encoding header, which
  453. # depends on how the server was configured:
  454. # - Some servers will notice that the file isn't a
  455. # compressible file and will leave the file alone
  456. # and with an empty Content-Encoding
  457. # - Some servers will notice that the file is
  458. # already compressed and will leave the file
  459. # alone and will add a Content-Encoding: gzip
  460. # header
  461. # - Some servers won't notice anything at all and
  462. # will take a file that's already been compressed
  463. # and compress it again and set the
  464. # Content-Encoding: gzip header
  465. #
  466. # By setting this not to decode automatically we
  467. # hope to eliminate problems with the second case.
  468. decode_content=False):
  469. yield chunk
  470. except AttributeError:
  471. # Standard file-like object.
  472. while True:
  473. chunk = resp.raw.read(chunk_size)
  474. if not chunk:
  475. break
  476. yield chunk
  477. progress_indicator = _progress_indicator
  478. if link.netloc == PyPI.netloc:
  479. url = show_url
  480. else:
  481. url = link.url_without_fragment
  482. if show_progress: # We don't show progress on cached responses
  483. if total_length:
  484. logger.info(
  485. "Downloading %s (%s)", url, format_size(total_length),
  486. )
  487. progress_indicator = DownloadProgressBar(
  488. max=total_length,
  489. ).iter
  490. else:
  491. logger.info("Downloading %s", url)
  492. progress_indicator = DownloadProgressSpinner().iter
  493. elif cached_resp:
  494. logger.info("Using cached %s", url)
  495. else:
  496. logger.info("Downloading %s", url)
  497. logger.debug('Downloading from URL %s', link)
  498. for chunk in progress_indicator(resp_read(4096), 4096):
  499. if download_hash is not None:
  500. download_hash.update(chunk)
  501. content_file.write(chunk)
  502. if link.hash and link.hash_name:
  503. _check_hash(download_hash, link)
  504. return download_hash
  505. def _copy_file(filename, location, content_type, link):
  506. copy = True
  507. download_location = os.path.join(location, link.filename)
  508. if os.path.exists(download_location):
  509. response = ask_path_exists(
  510. 'The file %s exists. (i)gnore, (w)ipe, (b)ackup ' %
  511. display_path(download_location), ('i', 'w', 'b'))
  512. if response == 'i':
  513. copy = False
  514. elif response == 'w':
  515. logger.warning('Deleting %s', display_path(download_location))
  516. os.remove(download_location)
  517. elif response == 'b':
  518. dest_file = backup_dir(download_location)
  519. logger.warning(
  520. 'Backing up %s to %s',
  521. display_path(download_location),
  522. display_path(dest_file),
  523. )
  524. shutil.move(download_location, dest_file)
  525. if copy:
  526. shutil.copy(filename, download_location)
  527. logger.info('Saved %s', display_path(download_location))
  528. def unpack_http_url(link, location, download_dir=None, session=None):
  529. if session is None:
  530. raise TypeError(
  531. "unpack_http_url() missing 1 required keyword argument: 'session'"
  532. )
  533. temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
  534. # If a download dir is specified, is the file already downloaded there?
  535. already_downloaded_path = None
  536. if download_dir:
  537. already_downloaded_path = _check_download_dir(link, download_dir)
  538. if already_downloaded_path:
  539. from_path = already_downloaded_path
  540. content_type = mimetypes.guess_type(from_path)[0]
  541. else:
  542. # let's download to a tmp dir
  543. from_path, content_type = _download_http_url(link, session, temp_dir)
  544. # unpack the archive to the build dir location. even when only downloading
  545. # archives, they have to be unpacked to parse dependencies
  546. unpack_file(from_path, location, content_type, link)
  547. # a download dir is specified; let's copy the archive there
  548. if download_dir and not already_downloaded_path:
  549. _copy_file(from_path, download_dir, content_type, link)
  550. if not already_downloaded_path:
  551. os.unlink(from_path)
  552. rmtree(temp_dir)
  553. def unpack_file_url(link, location, download_dir=None):
  554. """Unpack link into location.
  555. If download_dir is provided and link points to a file, make a copy
  556. of the link file inside download_dir."""
  557. link_path = url_to_path(link.url_without_fragment)
  558. # If it's a url to a local directory
  559. if os.path.isdir(link_path):
  560. if os.path.isdir(location):
  561. rmtree(location)
  562. shutil.copytree(link_path, location, symlinks=True)
  563. if download_dir:
  564. logger.info('Link is a directory, ignoring download_dir')
  565. return
  566. # if link has a hash, let's confirm it matches
  567. if link.hash:
  568. link_path_hash = _get_hash_from_file(link_path, link)
  569. _check_hash(link_path_hash, link)
  570. # If a download dir is specified, is the file already there and valid?
  571. already_downloaded_path = None
  572. if download_dir:
  573. already_downloaded_path = _check_download_dir(link, download_dir)
  574. if already_downloaded_path:
  575. from_path = already_downloaded_path
  576. else:
  577. from_path = link_path
  578. content_type = mimetypes.guess_type(from_path)[0]
  579. # unpack the archive to the build dir location. even when only downloading
  580. # archives, they have to be unpacked to parse dependencies
  581. unpack_file(from_path, location, content_type, link)
  582. # a download dir is specified and not already downloaded
  583. if download_dir and not already_downloaded_path:
  584. _copy_file(from_path, download_dir, content_type, link)
  585. def _copy_dist_from_dir(link_path, location):
  586. """Copy distribution files in `link_path` to `location`.
  587. Invoked when user requests to install a local directory. E.g.:
  588. pip install .
  589. pip install ~/dev/git-repos/python-prompt-toolkit
  590. """
  591. # Note: This is currently VERY SLOW if you have a lot of data in the
  592. # directory, because it copies everything with `shutil.copytree`.
  593. # What it should really do is build an sdist and install that.
  594. # See https://github.com/pypa/pip/issues/2195
  595. if os.path.isdir(location):
  596. rmtree(location)
  597. # build an sdist
  598. setup_py = 'setup.py'
  599. sdist_args = [sys.executable]
  600. sdist_args.append('-c')
  601. sdist_args.append(
  602. "import setuptools, tokenize;__file__=%r;"
  603. "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
  604. ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py)
  605. sdist_args.append('sdist')
  606. sdist_args += ['--dist-dir', location]
  607. logger.info('Running setup.py sdist for %s', link_path)
  608. with indent_log():
  609. call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
  610. # unpack sdist into `location`
  611. sdist = os.path.join(location, os.listdir(location)[0])
  612. logger.info('Unpacking sdist %s into %s', sdist, location)
  613. unpack_file(sdist, location, content_type=None, link=None)
  614. class PipXmlrpcTransport(xmlrpc_client.Transport):
  615. """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
  616. object.
  617. """
  618. def __init__(self, index_url, session, use_datetime=False):
  619. xmlrpc_client.Transport.__init__(self, use_datetime)
  620. index_parts = urllib_parse.urlparse(index_url)
  621. self._scheme = index_parts.scheme
  622. self._session = session
  623. def request(self, host, handler, request_body, verbose=False):
  624. parts = (self._scheme, host, handler, None, None, None)
  625. url = urllib_parse.urlunparse(parts)
  626. try:
  627. headers = {'Content-Type': 'text/xml'}
  628. response = self._session.post(url, data=request_body,
  629. headers=headers, stream=True)
  630. response.raise_for_status()
  631. self.verbose = verbose
  632. return self.parse_response(response.raw)
  633. except requests.HTTPError as exc:
  634. logger.critical(
  635. "HTTP error %s while getting %s",
  636. exc.response.status_code, url,
  637. )
  638. raise
  639. def unpack_url(link, location, download_dir=None,
  640. only_download=False, session=None):
  641. """Unpack link.
  642. If link is a VCS link:
  643. if only_download, export into download_dir and ignore location
  644. else unpack into location
  645. for other types of link:
  646. - unpack into location
  647. - if download_dir, copy the file into download_dir
  648. - if only_download, mark location for deletion
  649. """
  650. # non-editable vcs urls
  651. if is_vcs_url(link):
  652. unpack_vcs_link(link, location)
  653. # file urls
  654. elif is_file_url(link):
  655. unpack_file_url(link, location, download_dir)
  656. # http urls
  657. else:
  658. if session is None:
  659. session = PipSession()
  660. unpack_http_url(
  661. link,
  662. location,
  663. download_dir,
  664. session,
  665. )
  666. if only_download:
  667. write_delete_marker_file(location)
  668. def _download_http_url(link, session, temp_dir):
  669. """Download link url into temp_dir using provided session"""
  670. target_url = link.url.split('#', 1)[0]
  671. try:
  672. resp = session.get(
  673. target_url,
  674. # We use Accept-Encoding: identity here because requests
  675. # defaults to accepting compressed responses. This breaks in
  676. # a variety of ways depending on how the server is configured.
  677. # - Some servers will notice that the file isn't a compressible
  678. # file and will leave the file alone and with an empty
  679. # Content-Encoding
  680. # - Some servers will notice that the file is already
  681. # compressed and will leave the file alone and will add a
  682. # Content-Encoding: gzip header
  683. # - Some servers won't notice anything at all and will take
  684. # a file that's already been compressed and compress it again
  685. # and set the Content-Encoding: gzip header
  686. # By setting this to request only the identity encoding We're
  687. # hoping to eliminate the third case. Hopefully there does not
  688. # exist a server which when given a file will notice it is
  689. # already compressed and that you're not asking for a
  690. # compressed file and will then decompress it before sending
  691. # because if that's the case I don't think it'll ever be
  692. # possible to make this work.
  693. headers={"Accept-Encoding": "identity"},
  694. stream=True,
  695. )
  696. resp.raise_for_status()
  697. except requests.HTTPError as exc:
  698. logger.critical(
  699. "HTTP error %s while getting %s", exc.response.status_code, link,
  700. )
  701. raise
  702. content_type = resp.headers.get('content-type', '')
  703. filename = link.filename # fallback
  704. # Have a look at the Content-Disposition header for a better guess
  705. content_disposition = resp.headers.get('content-disposition')
  706. if content_disposition:
  707. type, params = cgi.parse_header(content_disposition)
  708. # We use ``or`` here because we don't want to use an "empty" value
  709. # from the filename param.
  710. filename = params.get('filename') or filename
  711. ext = splitext(filename)[1]
  712. if not ext:
  713. ext = mimetypes.guess_extension(content_type)
  714. if ext:
  715. filename += ext
  716. if not ext and link.url != resp.url:
  717. ext = os.path.splitext(resp.url)[1]
  718. if ext:
  719. filename += ext
  720. file_path = os.path.join(temp_dir, filename)
  721. with open(file_path, 'wb') as content_file:
  722. _download_url(resp, link, content_file)
  723. return file_path, content_type
  724. def _check_download_dir(link, download_dir):
  725. """ Check download_dir for previously downloaded file with correct hash
  726. If a correct file is found return its path else None
  727. """
  728. download_path = os.path.join(download_dir, link.filename)
  729. if os.path.exists(download_path):
  730. # If already downloaded, does its hash match?
  731. logger.info('File was already downloaded %s', download_path)
  732. if link.hash:
  733. download_hash = _get_hash_from_file(download_path, link)
  734. try:
  735. _check_hash(download_hash, link)
  736. except HashMismatch:
  737. logger.warning(
  738. 'Previously-downloaded file %s has bad hash, '
  739. 're-downloading.',
  740. download_path
  741. )
  742. os.unlink(download_path)
  743. return None
  744. return download_path
  745. return None