You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

index.py 47 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299
  1. """Routines related to PyPI, indexes"""
  2. from __future__ import absolute_import
  3. import logging
  4. import cgi
  5. from collections import namedtuple
  6. import itertools
  7. import sys
  8. import os
  9. import re
  10. import mimetypes
  11. import posixpath
  12. import warnings
  13. from pip._vendor.six.moves.urllib import parse as urllib_parse
  14. from pip._vendor.six.moves.urllib import request as urllib_request
  15. from pip.compat import ipaddress
  16. from pip.utils import (
  17. Inf, cached_property, normalize_name, splitext, normalize_path,
  18. ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS)
  19. from pip.utils.deprecation import RemovedInPip8Warning
  20. from pip.utils.logging import indent_log
  21. from pip.exceptions import (
  22. DistributionNotFound, BestVersionAlreadyInstalled, InvalidWheelFilename,
  23. UnsupportedWheel,
  24. )
  25. from pip.download import HAS_TLS, url_to_path, path_to_url
  26. from pip.models import PyPI
  27. from pip.wheel import Wheel, wheel_ext
  28. from pip.pep425tags import supported_tags, supported_tags_noarch, get_platform
  29. from pip._vendor import html5lib, requests, pkg_resources, six
  30. from pip._vendor.packaging.version import parse as parse_version
  31. from pip._vendor.requests.exceptions import SSLError
  32. __all__ = ['FormatControl', 'fmt_ctl_handle_mutual_exclude', 'PackageFinder']
  33. # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
  34. SECURE_ORIGINS = [
  35. # protocol, hostname, port
  36. ("https", "*", "*"),
  37. ("*", "localhost", "*"),
  38. ("*", "127.0.0.0/8", "*"),
  39. ("*", "::1/128", "*"),
  40. ("file", "*", None),
  41. ]
  42. logger = logging.getLogger(__name__)
  43. class InstallationCandidate(object):
  44. def __init__(self, project, version, location):
  45. self.project = project
  46. self.version = parse_version(version)
  47. self.location = location
  48. self._key = (self.project, self.version, self.location)
  49. def __repr__(self):
  50. return "<InstallationCandidate({0!r}, {1!r}, {2!r})>".format(
  51. self.project, self.version, self.location,
  52. )
  53. def __hash__(self):
  54. return hash(self._key)
  55. def __lt__(self, other):
  56. return self._compare(other, lambda s, o: s < o)
  57. def __le__(self, other):
  58. return self._compare(other, lambda s, o: s <= o)
  59. def __eq__(self, other):
  60. return self._compare(other, lambda s, o: s == o)
  61. def __ge__(self, other):
  62. return self._compare(other, lambda s, o: s >= o)
  63. def __gt__(self, other):
  64. return self._compare(other, lambda s, o: s > o)
  65. def __ne__(self, other):
  66. return self._compare(other, lambda s, o: s != o)
  67. def _compare(self, other, method):
  68. if not isinstance(other, InstallationCandidate):
  69. return NotImplemented
  70. return method(self._key, other._key)
  71. class PackageFinder(object):
  72. """This finds packages.
  73. This is meant to match easy_install's technique for looking for
  74. packages, by reading pages and looking for appropriate links.
  75. """
  76. def __init__(self, find_links, index_urls,
  77. allow_external=(), allow_unverified=(),
  78. allow_all_external=False, allow_all_prereleases=False,
  79. trusted_hosts=None, process_dependency_links=False,
  80. session=None, format_control=None):
  81. """Create a PackageFinder.
  82. :param format_control: A FormatControl object or None. Used to control
  83. the selection of source packages / binary packages when consulting
  84. the index and links.
  85. """
  86. if session is None:
  87. raise TypeError(
  88. "PackageFinder() missing 1 required keyword argument: "
  89. "'session'"
  90. )
  91. # Build find_links. If an argument starts with ~, it may be
  92. # a local file relative to a home directory. So try normalizing
  93. # it and if it exists, use the normalized version.
  94. # This is deliberately conservative - it might be fine just to
  95. # blindly normalize anything starting with a ~...
  96. self.find_links = []
  97. for link in find_links:
  98. if link.startswith('~'):
  99. new_link = normalize_path(link)
  100. if os.path.exists(new_link):
  101. link = new_link
  102. self.find_links.append(link)
  103. self.index_urls = index_urls
  104. self.dependency_links = []
  105. # These are boring links that have already been logged somehow:
  106. self.logged_links = set()
  107. self.format_control = format_control or FormatControl(set(), set())
  108. # Do we allow (safe and verifiable) externally hosted files?
  109. self.allow_external = set(normalize_name(n) for n in allow_external)
  110. # Which names are allowed to install insecure and unverifiable files?
  111. self.allow_unverified = set(
  112. normalize_name(n) for n in allow_unverified
  113. )
  114. # Anything that is allowed unverified is also allowed external
  115. self.allow_external |= self.allow_unverified
  116. # Do we allow all (safe and verifiable) externally hosted files?
  117. self.allow_all_external = allow_all_external
  118. # Domains that we won't emit warnings for when not using HTTPS
  119. self.secure_origins = [
  120. ("*", host, "*")
  121. for host in (trusted_hosts if trusted_hosts else [])
  122. ]
  123. # Stores if we ignored any external links so that we can instruct
  124. # end users how to install them if no distributions are available
  125. self.need_warn_external = False
  126. # Stores if we ignored any unsafe links so that we can instruct
  127. # end users how to install them if no distributions are available
  128. self.need_warn_unverified = False
  129. # Do we want to allow _all_ pre-releases?
  130. self.allow_all_prereleases = allow_all_prereleases
  131. # Do we process dependency links?
  132. self.process_dependency_links = process_dependency_links
  133. # The Session we'll use to make requests
  134. self.session = session
  135. # If we don't have TLS enabled, then WARN if anyplace we're looking
  136. # relies on TLS.
  137. if not HAS_TLS:
  138. for link in itertools.chain(self.index_urls, self.find_links):
  139. parsed = urllib_parse.urlparse(link)
  140. if parsed.scheme == "https":
  141. logger.warning(
  142. "pip is configured with locations that require "
  143. "TLS/SSL, however the ssl module in Python is not "
  144. "available."
  145. )
  146. break
  147. def add_dependency_links(self, links):
  148. # # FIXME: this shouldn't be global list this, it should only
  149. # # apply to requirements of the package that specifies the
  150. # # dependency_links value
  151. # # FIXME: also, we should track comes_from (i.e., use Link)
  152. if self.process_dependency_links:
  153. warnings.warn(
  154. "Dependency Links processing has been deprecated and will be "
  155. "removed in a future release.",
  156. RemovedInPip8Warning,
  157. )
  158. self.dependency_links.extend(links)
  159. @staticmethod
  160. def _sort_locations(locations, expand_dir=False):
  161. """
  162. Sort locations into "files" (archives) and "urls", and return
  163. a pair of lists (files,urls)
  164. """
  165. files = []
  166. urls = []
  167. # puts the url for the given file path into the appropriate list
  168. def sort_path(path):
  169. url = path_to_url(path)
  170. if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
  171. urls.append(url)
  172. else:
  173. files.append(url)
  174. for url in locations:
  175. is_local_path = os.path.exists(url)
  176. is_file_url = url.startswith('file:')
  177. if is_local_path or is_file_url:
  178. if is_local_path:
  179. path = url
  180. else:
  181. path = url_to_path(url)
  182. if os.path.isdir(path):
  183. if expand_dir:
  184. path = os.path.realpath(path)
  185. for item in os.listdir(path):
  186. sort_path(os.path.join(path, item))
  187. elif is_file_url:
  188. urls.append(url)
  189. elif os.path.isfile(path):
  190. sort_path(path)
  191. else:
  192. urls.append(url)
  193. return files, urls
  194. def _candidate_sort_key(self, candidate):
  195. """
  196. Function used to generate link sort key for link tuples.
  197. The greater the return value, the more preferred it is.
  198. If not finding wheels, then sorted by version only.
  199. If finding wheels, then the sort order is by version, then:
  200. 1. existing installs
  201. 2. wheels ordered via Wheel.support_index_min()
  202. 3. source archives
  203. Note: it was considered to embed this logic into the Link
  204. comparison operators, but then different sdist links
  205. with the same version, would have to be considered equal
  206. """
  207. support_num = len(supported_tags)
  208. if candidate.location == INSTALLED_VERSION:
  209. pri = 1
  210. elif candidate.location.is_wheel:
  211. # can raise InvalidWheelFilename
  212. wheel = Wheel(candidate.location.filename)
  213. if not wheel.supported():
  214. raise UnsupportedWheel(
  215. "%s is not a supported wheel for this platform. It "
  216. "can't be sorted." % wheel.filename
  217. )
  218. pri = -(wheel.support_index_min())
  219. else: # sdist
  220. pri = -(support_num)
  221. return (candidate.version, pri)
  222. def _sort_versions(self, applicable_versions):
  223. """
  224. Bring the latest version (and wheels) to the front, but maintain the
  225. existing ordering as secondary. See the docstring for `_link_sort_key`
  226. for details. This function is isolated for easier unit testing.
  227. """
  228. return sorted(
  229. applicable_versions,
  230. key=self._candidate_sort_key,
  231. reverse=True
  232. )
  233. def _validate_secure_origin(self, logger, location):
  234. # Determine if this url used a secure transport mechanism
  235. parsed = urllib_parse.urlparse(str(location))
  236. origin = (parsed.scheme, parsed.hostname, parsed.port)
  237. # Determine if our origin is a secure origin by looking through our
  238. # hardcoded list of secure origins, as well as any additional ones
  239. # configured on this PackageFinder instance.
  240. for secure_origin in (SECURE_ORIGINS + self.secure_origins):
  241. # Check to see if the protocol matches
  242. if origin[0] != secure_origin[0] and secure_origin[0] != "*":
  243. continue
  244. try:
  245. # We need to do this decode dance to ensure that we have a
  246. # unicode object, even on Python 2.x.
  247. addr = ipaddress.ip_address(
  248. origin[1]
  249. if (
  250. isinstance(origin[1], six.text_type) or
  251. origin[1] is None
  252. )
  253. else origin[1].decode("utf8")
  254. )
  255. network = ipaddress.ip_network(
  256. secure_origin[1]
  257. if isinstance(secure_origin[1], six.text_type)
  258. else secure_origin[1].decode("utf8")
  259. )
  260. except ValueError:
  261. # We don't have both a valid address or a valid network, so
  262. # we'll check this origin against hostnames.
  263. if origin[1] != secure_origin[1] and secure_origin[1] != "*":
  264. continue
  265. else:
  266. # We have a valid address and network, so see if the address
  267. # is contained within the network.
  268. if addr not in network:
  269. continue
  270. # Check to see if the port patches
  271. if (origin[2] != secure_origin[2] and
  272. secure_origin[2] != "*" and
  273. secure_origin[2] is not None):
  274. continue
  275. # If we've gotten here, then this origin matches the current
  276. # secure origin and we should return True
  277. return True
  278. # If we've gotten to this point, then the origin isn't secure and we
  279. # will not accept it as a valid location to search. We will however
  280. # log a warning that we are ignoring it.
  281. logger.warning(
  282. "The repository located at %s is not a trusted or secure host and "
  283. "is being ignored. If this repository is available via HTTPS it "
  284. "is recommended to use HTTPS instead, otherwise you may silence "
  285. "this warning and allow it anyways with '--trusted-host %s'.",
  286. parsed.hostname,
  287. parsed.hostname,
  288. )
  289. return False
  290. def _get_index_urls_locations(self, project_name):
  291. """Returns the locations found via self.index_urls
  292. Checks the url_name on the main (first in the list) index and
  293. use this url_name to produce all locations
  294. """
  295. def mkurl_pypi_url(url):
  296. loc = posixpath.join(url, project_url_name)
  297. # For maximum compatibility with easy_install, ensure the path
  298. # ends in a trailing slash. Although this isn't in the spec
  299. # (and PyPI can handle it without the slash) some other index
  300. # implementations might break if they relied on easy_install's
  301. # behavior.
  302. if not loc.endswith('/'):
  303. loc = loc + '/'
  304. return loc
  305. project_url_name = urllib_parse.quote(project_name.lower())
  306. if self.index_urls:
  307. # Check that we have the url_name correctly spelled:
  308. # Only check main index if index URL is given
  309. main_index_url = Link(
  310. mkurl_pypi_url(self.index_urls[0]),
  311. trusted=True,
  312. )
  313. page = self._get_page(main_index_url)
  314. if page is None and PyPI.netloc not in str(main_index_url):
  315. warnings.warn(
  316. "Failed to find %r at %s. It is suggested to upgrade "
  317. "your index to support normalized names as the name in "
  318. "/simple/{name}." % (project_name, main_index_url),
  319. RemovedInPip8Warning,
  320. )
  321. project_url_name = self._find_url_name(
  322. Link(self.index_urls[0], trusted=True),
  323. project_url_name,
  324. ) or project_url_name
  325. if project_url_name is not None:
  326. return [mkurl_pypi_url(url) for url in self.index_urls]
  327. return []
  328. def _find_all_versions(self, project_name):
  329. """Find all available versions for project_name
  330. This checks index_urls, find_links and dependency_links
  331. All versions found are returned
  332. See _link_package_versions for details on which files are accepted
  333. """
  334. index_locations = self._get_index_urls_locations(project_name)
  335. index_file_loc, index_url_loc = self._sort_locations(index_locations)
  336. fl_file_loc, fl_url_loc = self._sort_locations(
  337. self.find_links, expand_dir=True)
  338. dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links)
  339. file_locations = (
  340. Link(url) for url in itertools.chain(
  341. index_file_loc, fl_file_loc, dep_file_loc)
  342. )
  343. # We trust every url that the user has given us whether it was given
  344. # via --index-url or --find-links
  345. # We explicitly do not trust links that came from dependency_links
  346. # We want to filter out any thing which does not have a secure origin.
  347. url_locations = [
  348. link for link in itertools.chain(
  349. (Link(url, trusted=True) for url in index_url_loc),
  350. (Link(url, trusted=True) for url in fl_url_loc),
  351. (Link(url) for url in dep_url_loc),
  352. )
  353. if self._validate_secure_origin(logger, link)
  354. ]
  355. logger.debug('%d location(s) to search for versions of %s:',
  356. len(url_locations), project_name)
  357. for location in url_locations:
  358. logger.debug('* %s', location)
  359. canonical_name = pkg_resources.safe_name(project_name).lower()
  360. formats = fmt_ctl_formats(self.format_control, canonical_name)
  361. search = Search(project_name.lower(), canonical_name, formats)
  362. find_links_versions = self._package_versions(
  363. # We trust every directly linked archive in find_links
  364. (Link(url, '-f', trusted=True) for url in self.find_links),
  365. search
  366. )
  367. page_versions = []
  368. for page in self._get_pages(url_locations, project_name):
  369. logger.debug('Analyzing links from page %s', page.url)
  370. with indent_log():
  371. page_versions.extend(
  372. self._package_versions(page.links, search)
  373. )
  374. dependency_versions = self._package_versions(
  375. (Link(url) for url in self.dependency_links), search
  376. )
  377. if dependency_versions:
  378. logger.debug(
  379. 'dependency_links found: %s',
  380. ', '.join([
  381. version.location.url for version in dependency_versions
  382. ])
  383. )
  384. file_versions = self._package_versions(file_locations, search)
  385. if file_versions:
  386. file_versions.sort(reverse=True)
  387. logger.debug(
  388. 'Local files found: %s',
  389. ', '.join([
  390. url_to_path(candidate.location.url)
  391. for candidate in file_versions
  392. ])
  393. )
  394. # This is an intentional priority ordering
  395. return (
  396. file_versions + find_links_versions + page_versions +
  397. dependency_versions
  398. )
  399. def find_requirement(self, req, upgrade):
  400. """Try to find an InstallationCandidate for req
  401. Expects req, an InstallRequirement and upgrade, a boolean
  402. Returns an InstallationCandidate or None
  403. May raise DistributionNotFound or BestVersionAlreadyInstalled
  404. """
  405. all_versions = self._find_all_versions(req.name)
  406. # Filter out anything which doesn't match our specifier
  407. _versions = set(
  408. req.specifier.filter(
  409. # We turn the version object into a str here because otherwise
  410. # when we're debundled but setuptools isn't, Python will see
  411. # packaging.version.Version and
  412. # pkg_resources._vendor.packaging.version.Version as different
  413. # types. This way we'll use a str as a common data interchange
  414. # format. If we stop using the pkg_resources provided specifier
  415. # and start using our own, we can drop the cast to str().
  416. [str(x.version) for x in all_versions],
  417. prereleases=(
  418. self.allow_all_prereleases
  419. if self.allow_all_prereleases else None
  420. ),
  421. )
  422. )
  423. applicable_versions = [
  424. # Again, converting to str to deal with debundling.
  425. x for x in all_versions if str(x.version) in _versions
  426. ]
  427. if req.satisfied_by is not None:
  428. # Finally add our existing versions to the front of our versions.
  429. applicable_versions.insert(
  430. 0,
  431. InstallationCandidate(
  432. req.name,
  433. req.satisfied_by.version,
  434. INSTALLED_VERSION,
  435. )
  436. )
  437. existing_applicable = True
  438. else:
  439. existing_applicable = False
  440. applicable_versions = self._sort_versions(applicable_versions)
  441. if not upgrade and existing_applicable:
  442. if applicable_versions[0].location is INSTALLED_VERSION:
  443. logger.debug(
  444. 'Existing installed version (%s) is most up-to-date and '
  445. 'satisfies requirement',
  446. req.satisfied_by.version,
  447. )
  448. else:
  449. logger.debug(
  450. 'Existing installed version (%s) satisfies requirement '
  451. '(most up-to-date version is %s)',
  452. req.satisfied_by.version,
  453. applicable_versions[0][2],
  454. )
  455. return None
  456. if not applicable_versions:
  457. logger.critical(
  458. 'Could not find a version that satisfies the requirement %s '
  459. '(from versions: %s)',
  460. req,
  461. ', '.join(
  462. sorted(
  463. set(str(i.version) for i in all_versions),
  464. key=parse_version,
  465. )
  466. )
  467. )
  468. if self.need_warn_external:
  469. logger.warning(
  470. "Some externally hosted files were ignored as access to "
  471. "them may be unreliable (use --allow-external %s to "
  472. "allow).",
  473. req.name,
  474. )
  475. if self.need_warn_unverified:
  476. logger.warning(
  477. "Some insecure and unverifiable files were ignored"
  478. " (use --allow-unverified %s to allow).",
  479. req.name,
  480. )
  481. raise DistributionNotFound(
  482. 'No matching distribution found for %s' % req
  483. )
  484. if applicable_versions[0].location is INSTALLED_VERSION:
  485. # We have an existing version, and its the best version
  486. logger.debug(
  487. 'Installed version (%s) is most up-to-date (past versions: '
  488. '%s)',
  489. req.satisfied_by.version,
  490. ', '.join(str(i.version) for i in applicable_versions[1:]) or
  491. "none",
  492. )
  493. raise BestVersionAlreadyInstalled
  494. if len(applicable_versions) > 1:
  495. logger.debug(
  496. 'Using version %s (newest of versions: %s)',
  497. applicable_versions[0].version,
  498. ', '.join(str(i.version) for i in applicable_versions)
  499. )
  500. selected_version = applicable_versions[0].location
  501. if (selected_version.verifiable is not None and not
  502. selected_version.verifiable):
  503. logger.warning(
  504. "%s is potentially insecure and unverifiable.", req.name,
  505. )
  506. return selected_version
  507. def _find_url_name(self, index_url, url_name):
  508. """
  509. Finds the true URL name of a package, when the given name isn't quite
  510. correct.
  511. This is usually used to implement case-insensitivity.
  512. """
  513. if not index_url.url.endswith('/'):
  514. # Vaguely part of the PyPI API... weird but true.
  515. # FIXME: bad to modify this?
  516. index_url.url += '/'
  517. page = self._get_page(index_url)
  518. if page is None:
  519. logger.critical('Cannot fetch index base URL %s', index_url)
  520. return
  521. norm_name = normalize_name(url_name)
  522. for link in page.links:
  523. base = posixpath.basename(link.path.rstrip('/'))
  524. if norm_name == normalize_name(base):
  525. logger.debug(
  526. 'Real name of requirement %s is %s', url_name, base,
  527. )
  528. return base
  529. return None
  530. def _get_pages(self, locations, project_name):
  531. """
  532. Yields (page, page_url) from the given locations, skipping
  533. locations that have errors, and adding download/homepage links
  534. """
  535. all_locations = list(locations)
  536. seen = set()
  537. normalized = normalize_name(project_name)
  538. while all_locations:
  539. location = all_locations.pop(0)
  540. if location in seen:
  541. continue
  542. seen.add(location)
  543. page = self._get_page(location)
  544. if page is None:
  545. continue
  546. yield page
  547. for link in page.rel_links():
  548. if (normalized not in self.allow_external and not
  549. self.allow_all_external):
  550. self.need_warn_external = True
  551. logger.debug(
  552. "Not searching %s for files because external "
  553. "urls are disallowed.",
  554. link,
  555. )
  556. continue
  557. if (link.trusted is not None and not
  558. link.trusted and
  559. normalized not in self.allow_unverified):
  560. logger.debug(
  561. "Not searching %s for urls, it is an "
  562. "untrusted link and cannot produce safe or "
  563. "verifiable files.",
  564. link,
  565. )
  566. self.need_warn_unverified = True
  567. continue
  568. all_locations.append(link)
  569. _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
  570. def _sort_links(self, links):
  571. """
  572. Returns elements of links in order, non-egg links first, egg links
  573. second, while eliminating duplicates
  574. """
  575. eggs, no_eggs = [], []
  576. seen = set()
  577. for link in links:
  578. if link not in seen:
  579. seen.add(link)
  580. if link.egg_fragment:
  581. eggs.append(link)
  582. else:
  583. no_eggs.append(link)
  584. return no_eggs + eggs
  585. def _package_versions(self, links, search):
  586. result = []
  587. for link in self._sort_links(links):
  588. v = self._link_package_versions(link, search)
  589. if v is not None:
  590. result.append(v)
  591. return result
  592. def _log_skipped_link(self, link, reason):
  593. if link not in self.logged_links:
  594. logger.debug('Skipping link %s; %s', link, reason)
  595. self.logged_links.add(link)
  596. def _link_package_versions(self, link, search):
  597. """Return an InstallationCandidate or None"""
  598. platform = get_platform()
  599. version = None
  600. if link.egg_fragment:
  601. egg_info = link.egg_fragment
  602. ext = link.ext
  603. else:
  604. egg_info, ext = link.splitext()
  605. if not ext:
  606. self._log_skipped_link(link, 'not a file')
  607. return
  608. if ext not in SUPPORTED_EXTENSIONS:
  609. self._log_skipped_link(
  610. link, 'unsupported archive format: %s' % ext)
  611. return
  612. if "binary" not in search.formats and ext == wheel_ext:
  613. self._log_skipped_link(
  614. link, 'No binaries permitted for %s' % search.supplied)
  615. return
  616. if "macosx10" in link.path and ext == '.zip':
  617. self._log_skipped_link(link, 'macosx10 one')
  618. return
  619. if ext == wheel_ext:
  620. try:
  621. wheel = Wheel(link.filename)
  622. except InvalidWheelFilename:
  623. self._log_skipped_link(link, 'invalid wheel filename')
  624. return
  625. if (pkg_resources.safe_name(wheel.name).lower() !=
  626. search.canonical):
  627. self._log_skipped_link(
  628. link, 'wrong project name (not %s)' % search.supplied)
  629. return
  630. if not wheel.supported():
  631. self._log_skipped_link(
  632. link, 'it is not compatible with this Python')
  633. return
  634. # This is a dirty hack to prevent installing Binary Wheels from
  635. # PyPI unless it is a Windows or Mac Binary Wheel. This is
  636. # paired with a change to PyPI disabling uploads for the
  637. # same. Once we have a mechanism for enabling support for
  638. # binary wheels on linux that deals with the inherent problems
  639. # of binary distribution this can be removed.
  640. comes_from = getattr(link, "comes_from", None)
  641. if (
  642. (
  643. not platform.startswith('win') and not
  644. platform.startswith('macosx') and not
  645. platform == 'cli'
  646. ) and
  647. comes_from is not None and
  648. urllib_parse.urlparse(
  649. comes_from.url
  650. ).netloc.endswith(PyPI.netloc)):
  651. if not wheel.supported(tags=supported_tags_noarch):
  652. self._log_skipped_link(
  653. link,
  654. "it is a pypi-hosted binary "
  655. "Wheel on an unsupported platform",
  656. )
  657. return
  658. version = wheel.version
  659. # This should be up by the search.ok_binary check, but see issue 2700.
  660. if "source" not in search.formats and ext != wheel_ext:
  661. self._log_skipped_link(
  662. link, 'No sources permitted for %s' % search.supplied)
  663. return
  664. if not version:
  665. version = egg_info_matches(egg_info, search.supplied, link)
  666. if version is None:
  667. self._log_skipped_link(
  668. link, 'wrong project name (not %s)' % search.supplied)
  669. return
  670. if (link.internal is not None and not
  671. link.internal and not
  672. normalize_name(search.supplied).lower()
  673. in self.allow_external and not
  674. self.allow_all_external):
  675. # We have a link that we are sure is external, so we should skip
  676. # it unless we are allowing externals
  677. self._log_skipped_link(link, 'it is externally hosted')
  678. self.need_warn_external = True
  679. return
  680. if (link.verifiable is not None and not
  681. link.verifiable and not
  682. (normalize_name(search.supplied).lower()
  683. in self.allow_unverified)):
  684. # We have a link that we are sure we cannot verify its integrity,
  685. # so we should skip it unless we are allowing unsafe installs
  686. # for this requirement.
  687. self._log_skipped_link(
  688. link, 'it is an insecure and unverifiable file')
  689. self.need_warn_unverified = True
  690. return
  691. match = self._py_version_re.search(version)
  692. if match:
  693. version = version[:match.start()]
  694. py_version = match.group(1)
  695. if py_version != sys.version[:3]:
  696. self._log_skipped_link(
  697. link, 'Python version is incorrect')
  698. return
  699. logger.debug('Found link %s, version: %s', link, version)
  700. return InstallationCandidate(search.supplied, version, link)
  701. def _get_page(self, link):
  702. return HTMLPage.get_page(link, session=self.session)
  703. def egg_info_matches(
  704. egg_info, search_name, link,
  705. _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)):
  706. """Pull the version part out of a string.
  707. :param egg_info: The string to parse. E.g. foo-2.1
  708. :param search_name: The name of the package this belongs to. None to
  709. infer the name. Note that this cannot unambiguously parse strings
  710. like foo-2-2 which might be foo, 2-2 or foo-2, 2.
  711. :param link: The link the string came from, for logging on failure.
  712. """
  713. match = _egg_info_re.search(egg_info)
  714. if not match:
  715. logger.debug('Could not parse version from link: %s', link)
  716. return None
  717. if search_name is None:
  718. full_match = match.group(0)
  719. return full_match[full_match.index('-'):]
  720. name = match.group(0).lower()
  721. # To match the "safe" name that pkg_resources creates:
  722. name = name.replace('_', '-')
  723. # project name and version must be separated by a dash
  724. look_for = search_name.lower() + "-"
  725. if name.startswith(look_for):
  726. return match.group(0)[len(look_for):]
  727. else:
  728. return None
  729. class HTMLPage(object):
  730. """Represents one page, along with its URL"""
  731. def __init__(self, content, url, headers=None, trusted=None):
  732. # Determine if we have any encoding information in our headers
  733. encoding = None
  734. if headers and "Content-Type" in headers:
  735. content_type, params = cgi.parse_header(headers["Content-Type"])
  736. if "charset" in params:
  737. encoding = params['charset']
  738. self.content = content
  739. self.parsed = html5lib.parse(
  740. self.content,
  741. encoding=encoding,
  742. namespaceHTMLElements=False,
  743. )
  744. self.url = url
  745. self.headers = headers
  746. self.trusted = trusted
  747. def __str__(self):
  748. return self.url
  749. @classmethod
  750. def get_page(cls, link, skip_archives=True, session=None):
  751. if session is None:
  752. raise TypeError(
  753. "get_page() missing 1 required keyword argument: 'session'"
  754. )
  755. url = link.url
  756. url = url.split('#', 1)[0]
  757. # Check for VCS schemes that do not support lookup as web pages.
  758. from pip.vcs import VcsSupport
  759. for scheme in VcsSupport.schemes:
  760. if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
  761. logger.debug('Cannot look at %s URL %s', scheme, link)
  762. return None
  763. try:
  764. if skip_archives:
  765. filename = link.filename
  766. for bad_ext in ARCHIVE_EXTENSIONS:
  767. if filename.endswith(bad_ext):
  768. content_type = cls._get_content_type(
  769. url, session=session,
  770. )
  771. if content_type.lower().startswith('text/html'):
  772. break
  773. else:
  774. logger.debug(
  775. 'Skipping page %s because of Content-Type: %s',
  776. link,
  777. content_type,
  778. )
  779. return
  780. logger.debug('Getting page %s', url)
  781. # Tack index.html onto file:// URLs that point to directories
  782. (scheme, netloc, path, params, query, fragment) = \
  783. urllib_parse.urlparse(url)
  784. if (scheme == 'file' and
  785. os.path.isdir(urllib_request.url2pathname(path))):
  786. # add trailing slash if not present so urljoin doesn't trim
  787. # final segment
  788. if not url.endswith('/'):
  789. url += '/'
  790. url = urllib_parse.urljoin(url, 'index.html')
  791. logger.debug(' file: URL is directory, getting %s', url)
  792. resp = session.get(
  793. url,
  794. headers={
  795. "Accept": "text/html",
  796. "Cache-Control": "max-age=600",
  797. },
  798. )
  799. resp.raise_for_status()
  800. # The check for archives above only works if the url ends with
  801. # something that looks like an archive. However that is not a
  802. # requirement of an url. Unless we issue a HEAD request on every
  803. # url we cannot know ahead of time for sure if something is HTML
  804. # or not. However we can check after we've downloaded it.
  805. content_type = resp.headers.get('Content-Type', 'unknown')
  806. if not content_type.lower().startswith("text/html"):
  807. logger.debug(
  808. 'Skipping page %s because of Content-Type: %s',
  809. link,
  810. content_type,
  811. )
  812. return
  813. inst = cls(
  814. resp.content, resp.url, resp.headers,
  815. trusted=link.trusted,
  816. )
  817. except requests.HTTPError as exc:
  818. level = 2 if exc.response.status_code == 404 else 1
  819. cls._handle_fail(link, exc, url, level=level)
  820. except requests.ConnectionError as exc:
  821. cls._handle_fail(link, "connection error: %s" % exc, url)
  822. except requests.Timeout:
  823. cls._handle_fail(link, "timed out", url)
  824. except SSLError as exc:
  825. reason = ("There was a problem confirming the ssl certificate: "
  826. "%s" % exc)
  827. cls._handle_fail(link, reason, url, level=2, meth=logger.info)
  828. else:
  829. return inst
  830. @staticmethod
  831. def _handle_fail(link, reason, url, level=1, meth=None):
  832. if meth is None:
  833. meth = logger.debug
  834. meth("Could not fetch URL %s: %s - skipping", link, reason)
  835. @staticmethod
  836. def _get_content_type(url, session):
  837. """Get the Content-Type of the given url, using a HEAD request"""
  838. scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
  839. if scheme not in ('http', 'https'):
  840. # FIXME: some warning or something?
  841. # assertion error?
  842. return ''
  843. resp = session.head(url, allow_redirects=True)
  844. resp.raise_for_status()
  845. return resp.headers.get("Content-Type", "")
  846. @cached_property
  847. def api_version(self):
  848. metas = [
  849. x for x in self.parsed.findall(".//meta")
  850. if x.get("name", "").lower() == "api-version"
  851. ]
  852. if metas:
  853. try:
  854. return int(metas[0].get("value", None))
  855. except (TypeError, ValueError):
  856. pass
  857. return None
  858. @cached_property
  859. def base_url(self):
  860. bases = [
  861. x for x in self.parsed.findall(".//base")
  862. if x.get("href") is not None
  863. ]
  864. if bases and bases[0].get("href"):
  865. return bases[0].get("href")
  866. else:
  867. return self.url
  868. @property
  869. def links(self):
  870. """Yields all links in the page"""
  871. for anchor in self.parsed.findall(".//a"):
  872. if anchor.get("href"):
  873. href = anchor.get("href")
  874. url = self.clean_link(
  875. urllib_parse.urljoin(self.base_url, href)
  876. )
  877. # Determine if this link is internal. If that distinction
  878. # doesn't make sense in this context, then we don't make
  879. # any distinction.
  880. internal = None
  881. if self.api_version and self.api_version >= 2:
  882. # Only api_versions >= 2 have a distinction between
  883. # external and internal links
  884. internal = bool(
  885. anchor.get("rel") and
  886. "internal" in anchor.get("rel").split()
  887. )
  888. yield Link(url, self, internal=internal)
  889. def rel_links(self, rels=('homepage', 'download')):
  890. """Yields all links with the given relations"""
  891. rels = set(rels)
  892. for anchor in self.parsed.findall(".//a"):
  893. if anchor.get("rel") and anchor.get("href"):
  894. found_rels = set(anchor.get("rel").split())
  895. # Determine the intersection between what rels were found and
  896. # what rels were being looked for
  897. if found_rels & rels:
  898. href = anchor.get("href")
  899. url = self.clean_link(
  900. urllib_parse.urljoin(self.base_url, href)
  901. )
  902. yield Link(url, self, trusted=False)
  903. _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
  904. def clean_link(self, url):
  905. """Makes sure a link is fully encoded. That is, if a ' ' shows up in
  906. the link, it will be rewritten to %20 (while not over-quoting
  907. % or other characters)."""
  908. return self._clean_re.sub(
  909. lambda match: '%%%2x' % ord(match.group(0)), url)
  910. class Link(object):
  911. def __init__(self, url, comes_from=None, internal=None, trusted=None):
  912. # url can be a UNC windows share
  913. if url != Inf and url.startswith('\\\\'):
  914. url = path_to_url(url)
  915. self.url = url
  916. self.comes_from = comes_from
  917. self.internal = internal
  918. self.trusted = trusted
  919. def __str__(self):
  920. if self.comes_from:
  921. return '%s (from %s)' % (self.url, self.comes_from)
  922. else:
  923. return str(self.url)
  924. def __repr__(self):
  925. return '<Link %s>' % self
  926. def __eq__(self, other):
  927. if not isinstance(other, Link):
  928. return NotImplemented
  929. return self.url == other.url
  930. def __ne__(self, other):
  931. if not isinstance(other, Link):
  932. return NotImplemented
  933. return self.url != other.url
  934. def __lt__(self, other):
  935. if not isinstance(other, Link):
  936. return NotImplemented
  937. return self.url < other.url
  938. def __le__(self, other):
  939. if not isinstance(other, Link):
  940. return NotImplemented
  941. return self.url <= other.url
  942. def __gt__(self, other):
  943. if not isinstance(other, Link):
  944. return NotImplemented
  945. return self.url > other.url
  946. def __ge__(self, other):
  947. if not isinstance(other, Link):
  948. return NotImplemented
  949. return self.url >= other.url
  950. def __hash__(self):
  951. return hash(self.url)
  952. @property
  953. def filename(self):
  954. _, netloc, path, _, _ = urllib_parse.urlsplit(self.url)
  955. name = posixpath.basename(path.rstrip('/')) or netloc
  956. name = urllib_parse.unquote(name)
  957. assert name, ('URL %r produced no filename' % self.url)
  958. return name
  959. @property
  960. def scheme(self):
  961. return urllib_parse.urlsplit(self.url)[0]
  962. @property
  963. def netloc(self):
  964. return urllib_parse.urlsplit(self.url)[1]
  965. @property
  966. def path(self):
  967. return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2])
  968. def splitext(self):
  969. return splitext(posixpath.basename(self.path.rstrip('/')))
  970. @property
  971. def ext(self):
  972. return self.splitext()[1]
  973. @property
  974. def url_without_fragment(self):
  975. scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url)
  976. return urllib_parse.urlunsplit((scheme, netloc, path, query, None))
  977. _egg_fragment_re = re.compile(r'#egg=([^&]*)')
  978. @property
  979. def egg_fragment(self):
  980. match = self._egg_fragment_re.search(self.url)
  981. if not match:
  982. return None
  983. return match.group(1)
  984. _hash_re = re.compile(
  985. r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)'
  986. )
  987. @property
  988. def hash(self):
  989. match = self._hash_re.search(self.url)
  990. if match:
  991. return match.group(2)
  992. return None
  993. @property
  994. def hash_name(self):
  995. match = self._hash_re.search(self.url)
  996. if match:
  997. return match.group(1)
  998. return None
  999. @property
  1000. def show_url(self):
  1001. return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
  1002. @property
  1003. def verifiable(self):
  1004. """
  1005. Returns True if this link can be verified after download, False if it
  1006. cannot, and None if we cannot determine.
  1007. """
  1008. trusted = self.trusted or getattr(self.comes_from, "trusted", None)
  1009. if trusted is not None and trusted:
  1010. # This link came from a trusted source. It *may* be verifiable but
  1011. # first we need to see if this page is operating under the new
  1012. # API version.
  1013. try:
  1014. api_version = getattr(self.comes_from, "api_version", None)
  1015. api_version = int(api_version)
  1016. except (ValueError, TypeError):
  1017. api_version = None
  1018. if api_version is None or api_version <= 1:
  1019. # This link is either trusted, or it came from a trusted,
  1020. # however it is not operating under the API version 2 so
  1021. # we can't make any claims about if it's safe or not
  1022. return
  1023. if self.hash:
  1024. # This link came from a trusted source and it has a hash, so we
  1025. # can consider it safe.
  1026. return True
  1027. else:
  1028. # This link came from a trusted source, using the new API
  1029. # version, and it does not have a hash. It is NOT verifiable
  1030. return False
  1031. elif trusted is not None:
  1032. # This link came from an untrusted source and we cannot trust it
  1033. return False
  1034. @property
  1035. def is_wheel(self):
  1036. return self.ext == wheel_ext
  1037. @property
  1038. def is_artifact(self):
  1039. """
  1040. Determines if this points to an actual artifact (e.g. a tarball) or if
  1041. it points to an "abstract" thing like a path or a VCS location.
  1042. """
  1043. from pip.vcs import vcs
  1044. if self.scheme in vcs.all_schemes:
  1045. return False
  1046. return True
  1047. # An object to represent the "link" for the installed version of a requirement.
  1048. # Using Inf as the url makes it sort higher.
  1049. INSTALLED_VERSION = Link(Inf)
  1050. FormatControl = namedtuple('FormatControl', 'no_binary only_binary')
  1051. """This object has two fields, no_binary and only_binary.
  1052. If a field is falsy, it isn't set. If it is {':all:'}, it should match all
  1053. packages except those listed in the other field. Only one field can be set
  1054. to {':all:'} at a time. The rest of the time exact package name matches
  1055. are listed, with any given package only showing up in one field at a time.
  1056. """
  1057. def fmt_ctl_handle_mutual_exclude(value, target, other):
  1058. new = value.split(',')
  1059. while ':all:' in new:
  1060. other.clear()
  1061. target.clear()
  1062. target.add(':all:')
  1063. del new[:new.index(':all:') + 1]
  1064. if ':none:' not in new:
  1065. # Without a none, we want to discard everything as :all: covers it
  1066. return
  1067. for name in new:
  1068. if name == ':none:':
  1069. target.clear()
  1070. continue
  1071. name = pkg_resources.safe_name(name).lower()
  1072. other.discard(name)
  1073. target.add(name)
  1074. def fmt_ctl_formats(fmt_ctl, canonical_name):
  1075. result = set(["binary", "source"])
  1076. if canonical_name in fmt_ctl.only_binary:
  1077. result.discard('source')
  1078. elif canonical_name in fmt_ctl.no_binary:
  1079. result.discard('binary')
  1080. elif ':all:' in fmt_ctl.only_binary:
  1081. result.discard('source')
  1082. elif ':all:' in fmt_ctl.no_binary:
  1083. result.discard('binary')
  1084. return frozenset(result)
  1085. def fmt_ctl_no_binary(fmt_ctl):
  1086. fmt_ctl_handle_mutual_exclude(
  1087. ':all:', fmt_ctl.no_binary, fmt_ctl.only_binary)
  1088. def fmt_ctl_no_use_wheel(fmt_ctl):
  1089. fmt_ctl_no_binary(fmt_ctl)
  1090. warnings.warn(
  1091. '--no-use-wheel is deprecated and will be removed in the future. '
  1092. ' Please use --no-binary :all: instead.', DeprecationWarning,
  1093. stacklevel=2)
  1094. Search = namedtuple('Search', 'supplied canonical formats')
  1095. """Capture key aspects of a search.
  1096. :attribute supplied: The user supplied package.
  1097. :attribute canonical: The canonical package name.
  1098. :attribute formats: The formats allowed for this package. Should be a set
  1099. with 'binary' or 'source' or both in it.
  1100. """