You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

1050 lines
38 KiB

  1. """PyPI and direct package downloading"""
  2. import sys
  3. import os
  4. import re
  5. import shutil
  6. import socket
  7. import base64
  8. import hashlib
  9. from functools import wraps
  10. from pkg_resources import (
  11. CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST,
  12. require, Environment, find_distributions, safe_name, safe_version,
  13. to_filename, Requirement, DEVELOP_DIST,
  14. )
  15. from setuptools import ssl_support
  16. from distutils import log
  17. from distutils.errors import DistutilsError
  18. from setuptools.compat import (urllib2, httplib, StringIO, HTTPError,
  19. urlparse, urlunparse, unquote, splituser,
  20. url2pathname, name2codepoint,
  21. unichr, urljoin, urlsplit, urlunsplit,
  22. ConfigParser)
  23. from setuptools.compat import filterfalse
  24. from fnmatch import translate
  25. from setuptools.py26compat import strip_fragment
  26. from setuptools.py27compat import get_all_headers
  27. EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
  28. HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
  29. # this is here to fix emacs' cruddy broken syntax highlighting
  30. PYPI_MD5 = re.compile(
  31. '<a href="([^"#]+)">([^<]+)</a>\n\s+\\(<a (?:title="MD5 hash"\n\s+)'
  32. 'href="[^?]+\?:action=show_md5&amp;digest=([0-9a-f]{32})">md5</a>\\)'
  33. )
  34. URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
  35. EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()
  36. __all__ = [
  37. 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
  38. 'interpret_distro_name',
  39. ]
  40. _SOCKET_TIMEOUT = 15
  41. def parse_bdist_wininst(name):
  42. """Return (base,pyversion) or (None,None) for possible .exe name"""
  43. lower = name.lower()
  44. base, py_ver, plat = None, None, None
  45. if lower.endswith('.exe'):
  46. if lower.endswith('.win32.exe'):
  47. base = name[:-10]
  48. plat = 'win32'
  49. elif lower.startswith('.win32-py',-16):
  50. py_ver = name[-7:-4]
  51. base = name[:-16]
  52. plat = 'win32'
  53. elif lower.endswith('.win-amd64.exe'):
  54. base = name[:-14]
  55. plat = 'win-amd64'
  56. elif lower.startswith('.win-amd64-py',-20):
  57. py_ver = name[-7:-4]
  58. base = name[:-20]
  59. plat = 'win-amd64'
  60. return base,py_ver,plat
  61. def egg_info_for_url(url):
  62. scheme, server, path, parameters, query, fragment = urlparse(url)
  63. base = unquote(path.split('/')[-1])
  64. if server=='sourceforge.net' and base=='download': # XXX Yuck
  65. base = unquote(path.split('/')[-2])
  66. if '#' in base: base, fragment = base.split('#',1)
  67. return base,fragment
  68. def distros_for_url(url, metadata=None):
  69. """Yield egg or source distribution objects that might be found at a URL"""
  70. base, fragment = egg_info_for_url(url)
  71. for dist in distros_for_location(url, base, metadata): yield dist
  72. if fragment:
  73. match = EGG_FRAGMENT.match(fragment)
  74. if match:
  75. for dist in interpret_distro_name(
  76. url, match.group(1), metadata, precedence = CHECKOUT_DIST
  77. ):
  78. yield dist
  79. def distros_for_location(location, basename, metadata=None):
  80. """Yield egg or source distribution objects based on basename"""
  81. if basename.endswith('.egg.zip'):
  82. basename = basename[:-4] # strip the .zip
  83. if basename.endswith('.egg') and '-' in basename:
  84. # only one, unambiguous interpretation
  85. return [Distribution.from_location(location, basename, metadata)]
  86. if basename.endswith('.exe'):
  87. win_base, py_ver, platform = parse_bdist_wininst(basename)
  88. if win_base is not None:
  89. return interpret_distro_name(
  90. location, win_base, metadata, py_ver, BINARY_DIST, platform
  91. )
  92. # Try source distro extensions (.zip, .tgz, etc.)
  93. #
  94. for ext in EXTENSIONS:
  95. if basename.endswith(ext):
  96. basename = basename[:-len(ext)]
  97. return interpret_distro_name(location, basename, metadata)
  98. return [] # no extension matched
  99. def distros_for_filename(filename, metadata=None):
  100. """Yield possible egg or source distribution objects based on a filename"""
  101. return distros_for_location(
  102. normalize_path(filename), os.path.basename(filename), metadata
  103. )
  104. def interpret_distro_name(
  105. location, basename, metadata, py_version=None, precedence=SOURCE_DIST,
  106. platform=None
  107. ):
  108. """Generate alternative interpretations of a source distro name
  109. Note: if `location` is a filesystem filename, you should call
  110. ``pkg_resources.normalize_path()`` on it before passing it to this
  111. routine!
  112. """
  113. # Generate alternative interpretations of a source distro name
  114. # Because some packages are ambiguous as to name/versions split
  115. # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
  116. # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
  117. # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice,
  118. # the spurious interpretations should be ignored, because in the event
  119. # there's also an "adns" package, the spurious "python-1.1.0" version will
  120. # compare lower than any numeric version number, and is therefore unlikely
  121. # to match a request for it. It's still a potential problem, though, and
  122. # in the long run PyPI and the distutils should go for "safe" names and
  123. # versions in distribution archive names (sdist and bdist).
  124. parts = basename.split('-')
  125. if not py_version and any(re.match('py\d\.\d$', p) for p in parts[2:]):
  126. # it is a bdist_dumb, not an sdist -- bail out
  127. return
  128. for p in range(1,len(parts)+1):
  129. yield Distribution(
  130. location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
  131. py_version=py_version, precedence = precedence,
  132. platform = platform
  133. )
  134. # From Python 2.7 docs
  135. def unique_everseen(iterable, key=None):
  136. "List unique elements, preserving order. Remember all elements ever seen."
  137. # unique_everseen('AAAABBBCCDAABBB') --> A B C D
  138. # unique_everseen('ABBCcAD', str.lower) --> A B C D
  139. seen = set()
  140. seen_add = seen.add
  141. if key is None:
  142. for element in filterfalse(seen.__contains__, iterable):
  143. seen_add(element)
  144. yield element
  145. else:
  146. for element in iterable:
  147. k = key(element)
  148. if k not in seen:
  149. seen_add(k)
  150. yield element
  151. def unique_values(func):
  152. """
  153. Wrap a function returning an iterable such that the resulting iterable
  154. only ever yields unique items.
  155. """
  156. @wraps(func)
  157. def wrapper(*args, **kwargs):
  158. return unique_everseen(func(*args, **kwargs))
  159. return wrapper
  160. REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
  161. # this line is here to fix emacs' cruddy broken syntax highlighting
  162. @unique_values
  163. def find_external_links(url, page):
  164. """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""
  165. for match in REL.finditer(page):
  166. tag, rel = match.groups()
  167. rels = set(map(str.strip, rel.lower().split(',')))
  168. if 'homepage' in rels or 'download' in rels:
  169. for match in HREF.finditer(tag):
  170. yield urljoin(url, htmldecode(match.group(1)))
  171. for tag in ("<th>Home Page", "<th>Download URL"):
  172. pos = page.find(tag)
  173. if pos!=-1:
  174. match = HREF.search(page,pos)
  175. if match:
  176. yield urljoin(url, htmldecode(match.group(1)))
  177. user_agent = "Python-urllib/%s setuptools/%s" % (
  178. sys.version[:3], require('setuptools')[0].version
  179. )
  180. class ContentChecker(object):
  181. """
  182. A null content checker that defines the interface for checking content
  183. """
  184. def feed(self, block):
  185. """
  186. Feed a block of data to the hash.
  187. """
  188. return
  189. def is_valid(self):
  190. """
  191. Check the hash. Return False if validation fails.
  192. """
  193. return True
  194. def report(self, reporter, template):
  195. """
  196. Call reporter with information about the checker (hash name)
  197. substituted into the template.
  198. """
  199. return
  200. class HashChecker(ContentChecker):
  201. pattern = re.compile(
  202. r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)='
  203. r'(?P<expected>[a-f0-9]+)'
  204. )
  205. def __init__(self, hash_name, expected):
  206. self.hash_name = hash_name
  207. self.hash = hashlib.new(hash_name)
  208. self.expected = expected
  209. @classmethod
  210. def from_url(cls, url):
  211. "Construct a (possibly null) ContentChecker from a URL"
  212. fragment = urlparse(url)[-1]
  213. if not fragment:
  214. return ContentChecker()
  215. match = cls.pattern.search(fragment)
  216. if not match:
  217. return ContentChecker()
  218. return cls(**match.groupdict())
  219. def feed(self, block):
  220. self.hash.update(block)
  221. def is_valid(self):
  222. return self.hash.hexdigest() == self.expected
  223. def report(self, reporter, template):
  224. msg = template % self.hash_name
  225. return reporter(msg)
  226. class PackageIndex(Environment):
  227. """A distribution index that scans web pages for download URLs"""
  228. def __init__(
  229. self, index_url="https://pypi.python.org/simple", hosts=('*',),
  230. ca_bundle=None, verify_ssl=True, *args, **kw
  231. ):
  232. Environment.__init__(self,*args,**kw)
  233. self.index_url = index_url + "/"[:not index_url.endswith('/')]
  234. self.scanned_urls = {}
  235. self.fetched_urls = {}
  236. self.package_pages = {}
  237. self.allows = re.compile('|'.join(map(translate,hosts))).match
  238. self.to_scan = []
  239. if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()):
  240. self.opener = ssl_support.opener_for(ca_bundle)
  241. else: self.opener = urllib2.urlopen
  242. def process_url(self, url, retrieve=False):
  243. """Evaluate a URL as a possible download, and maybe retrieve it"""
  244. if url in self.scanned_urls and not retrieve:
  245. return
  246. self.scanned_urls[url] = True
  247. if not URL_SCHEME(url):
  248. self.process_filename(url)
  249. return
  250. else:
  251. dists = list(distros_for_url(url))
  252. if dists:
  253. if not self.url_ok(url):
  254. return
  255. self.debug("Found link: %s", url)
  256. if dists or not retrieve or url in self.fetched_urls:
  257. list(map(self.add, dists))
  258. return # don't need the actual page
  259. if not self.url_ok(url):
  260. self.fetched_urls[url] = True
  261. return
  262. self.info("Reading %s", url)
  263. self.fetched_urls[url] = True # prevent multiple fetch attempts
  264. f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url)
  265. if f is None: return
  266. self.fetched_urls[f.url] = True
  267. if 'html' not in f.headers.get('content-type', '').lower():
  268. f.close() # not html, we can't process it
  269. return
  270. base = f.url # handle redirects
  271. page = f.read()
  272. if not isinstance(page, str): # We are in Python 3 and got bytes. We want str.
  273. if isinstance(f, HTTPError):
  274. # Errors have no charset, assume latin1:
  275. charset = 'latin-1'
  276. else:
  277. charset = f.headers.get_param('charset') or 'latin-1'
  278. page = page.decode(charset, "ignore")
  279. f.close()
  280. for match in HREF.finditer(page):
  281. link = urljoin(base, htmldecode(match.group(1)))
  282. self.process_url(link)
  283. if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
  284. page = self.process_index(url, page)
  285. def process_filename(self, fn, nested=False):
  286. # process filenames or directories
  287. if not os.path.exists(fn):
  288. self.warn("Not found: %s", fn)
  289. return
  290. if os.path.isdir(fn) and not nested:
  291. path = os.path.realpath(fn)
  292. for item in os.listdir(path):
  293. self.process_filename(os.path.join(path,item), True)
  294. dists = distros_for_filename(fn)
  295. if dists:
  296. self.debug("Found: %s", fn)
  297. list(map(self.add, dists))
  298. def url_ok(self, url, fatal=False):
  299. s = URL_SCHEME(url)
  300. if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]):
  301. return True
  302. msg = ("\nNote: Bypassing %s (disallowed host; see "
  303. "http://bit.ly/1dg9ijs for details).\n")
  304. if fatal:
  305. raise DistutilsError(msg % url)
  306. else:
  307. self.warn(msg, url)
  308. def scan_egg_links(self, search_path):
  309. for item in search_path:
  310. if os.path.isdir(item):
  311. for entry in os.listdir(item):
  312. if entry.endswith('.egg-link'):
  313. self.scan_egg_link(item, entry)
  314. def scan_egg_link(self, path, entry):
  315. lines = [_f for _f in map(str.strip,
  316. open(os.path.join(path, entry))) if _f]
  317. if len(lines)==2:
  318. for dist in find_distributions(os.path.join(path, lines[0])):
  319. dist.location = os.path.join(path, *lines)
  320. dist.precedence = SOURCE_DIST
  321. self.add(dist)
  322. def process_index(self,url,page):
  323. """Process the contents of a PyPI page"""
  324. def scan(link):
  325. # Process a URL to see if it's for a package page
  326. if link.startswith(self.index_url):
  327. parts = list(map(
  328. unquote, link[len(self.index_url):].split('/')
  329. ))
  330. if len(parts)==2 and '#' not in parts[1]:
  331. # it's a package page, sanitize and index it
  332. pkg = safe_name(parts[0])
  333. ver = safe_version(parts[1])
  334. self.package_pages.setdefault(pkg.lower(),{})[link] = True
  335. return to_filename(pkg), to_filename(ver)
  336. return None, None
  337. # process an index page into the package-page index
  338. for match in HREF.finditer(page):
  339. try:
  340. scan(urljoin(url, htmldecode(match.group(1))))
  341. except ValueError:
  342. pass
  343. pkg, ver = scan(url) # ensure this page is in the page index
  344. if pkg:
  345. # process individual package page
  346. for new_url in find_external_links(url, page):
  347. # Process the found URL
  348. base, frag = egg_info_for_url(new_url)
  349. if base.endswith('.py') and not frag:
  350. if ver:
  351. new_url+='#egg=%s-%s' % (pkg,ver)
  352. else:
  353. self.need_version_info(url)
  354. self.scan_url(new_url)
  355. return PYPI_MD5.sub(
  356. lambda m: '<a href="%s#md5=%s">%s</a>' % m.group(1,3,2), page
  357. )
  358. else:
  359. return "" # no sense double-scanning non-package pages
  360. def need_version_info(self, url):
  361. self.scan_all(
  362. "Page at %s links to .py file(s) without version info; an index "
  363. "scan is required.", url
  364. )
  365. def scan_all(self, msg=None, *args):
  366. if self.index_url not in self.fetched_urls:
  367. if msg: self.warn(msg,*args)
  368. self.info(
  369. "Scanning index of all packages (this may take a while)"
  370. )
  371. self.scan_url(self.index_url)
  372. def find_packages(self, requirement):
  373. self.scan_url(self.index_url + requirement.unsafe_name+'/')
  374. if not self.package_pages.get(requirement.key):
  375. # Fall back to safe version of the name
  376. self.scan_url(self.index_url + requirement.project_name+'/')
  377. if not self.package_pages.get(requirement.key):
  378. # We couldn't find the target package, so search the index page too
  379. self.not_found_in_index(requirement)
  380. for url in list(self.package_pages.get(requirement.key,())):
  381. # scan each page that might be related to the desired package
  382. self.scan_url(url)
  383. def obtain(self, requirement, installer=None):
  384. self.prescan()
  385. self.find_packages(requirement)
  386. for dist in self[requirement.key]:
  387. if dist in requirement:
  388. return dist
  389. self.debug("%s does not match %s", requirement, dist)
  390. return super(PackageIndex, self).obtain(requirement,installer)
  391. def check_hash(self, checker, filename, tfp):
  392. """
  393. checker is a ContentChecker
  394. """
  395. checker.report(self.debug,
  396. "Validating %%s checksum for %s" % filename)
  397. if not checker.is_valid():
  398. tfp.close()
  399. os.unlink(filename)
  400. raise DistutilsError(
  401. "%s validation failed for %s; "
  402. "possible download problem?" % (
  403. checker.hash.name, os.path.basename(filename))
  404. )
  405. def add_find_links(self, urls):
  406. """Add `urls` to the list that will be prescanned for searches"""
  407. for url in urls:
  408. if (
  409. self.to_scan is None # if we have already "gone online"
  410. or not URL_SCHEME(url) # or it's a local file/directory
  411. or url.startswith('file:')
  412. or list(distros_for_url(url)) # or a direct package link
  413. ):
  414. # then go ahead and process it now
  415. self.scan_url(url)
  416. else:
  417. # otherwise, defer retrieval till later
  418. self.to_scan.append(url)
  419. def prescan(self):
  420. """Scan urls scheduled for prescanning (e.g. --find-links)"""
  421. if self.to_scan:
  422. list(map(self.scan_url, self.to_scan))
  423. self.to_scan = None # from now on, go ahead and process immediately
  424. def not_found_in_index(self, requirement):
  425. if self[requirement.key]: # we've seen at least one distro
  426. meth, msg = self.info, "Couldn't retrieve index page for %r"
  427. else: # no distros seen for this name, might be misspelled
  428. meth, msg = (self.warn,
  429. "Couldn't find index page for %r (maybe misspelled?)")
  430. meth(msg, requirement.unsafe_name)
  431. self.scan_all()
  432. def download(self, spec, tmpdir):
  433. """Locate and/or download `spec` to `tmpdir`, returning a local path
  434. `spec` may be a ``Requirement`` object, or a string containing a URL,
  435. an existing local filename, or a project/version requirement spec
  436. (i.e. the string form of a ``Requirement`` object). If it is the URL
  437. of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
  438. that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
  439. automatically created alongside the downloaded file.
  440. If `spec` is a ``Requirement`` object or a string containing a
  441. project/version requirement spec, this method returns the location of
  442. a matching distribution (possibly after downloading it to `tmpdir`).
  443. If `spec` is a locally existing file or directory name, it is simply
  444. returned unchanged. If `spec` is a URL, it is downloaded to a subpath
  445. of `tmpdir`, and the local filename is returned. Various errors may be
  446. raised if a problem occurs during downloading.
  447. """
  448. if not isinstance(spec,Requirement):
  449. scheme = URL_SCHEME(spec)
  450. if scheme:
  451. # It's a url, download it to tmpdir
  452. found = self._download_url(scheme.group(1), spec, tmpdir)
  453. base, fragment = egg_info_for_url(spec)
  454. if base.endswith('.py'):
  455. found = self.gen_setup(found,fragment,tmpdir)
  456. return found
  457. elif os.path.exists(spec):
  458. # Existing file or directory, just return it
  459. return spec
  460. else:
  461. try:
  462. spec = Requirement.parse(spec)
  463. except ValueError:
  464. raise DistutilsError(
  465. "Not a URL, existing file, or requirement spec: %r" %
  466. (spec,)
  467. )
  468. return getattr(self.fetch_distribution(spec, tmpdir),'location',None)
  469. def fetch_distribution(
  470. self, requirement, tmpdir, force_scan=False, source=False,
  471. develop_ok=False, local_index=None
  472. ):
  473. """Obtain a distribution suitable for fulfilling `requirement`
  474. `requirement` must be a ``pkg_resources.Requirement`` instance.
  475. If necessary, or if the `force_scan` flag is set, the requirement is
  476. searched for in the (online) package index as well as the locally
  477. installed packages. If a distribution matching `requirement` is found,
  478. the returned distribution's ``location`` is the value you would have
  479. gotten from calling the ``download()`` method with the matching
  480. distribution's URL or filename. If no matching distribution is found,
  481. ``None`` is returned.
  482. If the `source` flag is set, only source distributions and source
  483. checkout links will be considered. Unless the `develop_ok` flag is
  484. set, development and system eggs (i.e., those using the ``.egg-info``
  485. format) will be ignored.
  486. """
  487. # process a Requirement
  488. self.info("Searching for %s", requirement)
  489. skipped = {}
  490. dist = None
  491. def find(req, env=None):
  492. if env is None:
  493. env = self
  494. # Find a matching distribution; may be called more than once
  495. for dist in env[req.key]:
  496. if dist.precedence==DEVELOP_DIST and not develop_ok:
  497. if dist not in skipped:
  498. self.warn("Skipping development or system egg: %s",dist)
  499. skipped[dist] = 1
  500. continue
  501. if dist in req and (dist.precedence<=SOURCE_DIST or not source):
  502. return dist
  503. if force_scan:
  504. self.prescan()
  505. self.find_packages(requirement)
  506. dist = find(requirement)
  507. if local_index is not None:
  508. dist = dist or find(requirement, local_index)
  509. if dist is None:
  510. if self.to_scan is not None:
  511. self.prescan()
  512. dist = find(requirement)
  513. if dist is None and not force_scan:
  514. self.find_packages(requirement)
  515. dist = find(requirement)
  516. if dist is None:
  517. self.warn(
  518. "No local packages or download links found for %s%s",
  519. (source and "a source distribution of " or ""),
  520. requirement,
  521. )
  522. else:
  523. self.info("Best match: %s", dist)
  524. return dist.clone(location=self.download(dist.location, tmpdir))
  525. def fetch(self, requirement, tmpdir, force_scan=False, source=False):
  526. """Obtain a file suitable for fulfilling `requirement`
  527. DEPRECATED; use the ``fetch_distribution()`` method now instead. For
  528. backward compatibility, this routine is identical but returns the
  529. ``location`` of the downloaded distribution instead of a distribution
  530. object.
  531. """
  532. dist = self.fetch_distribution(requirement,tmpdir,force_scan,source)
  533. if dist is not None:
  534. return dist.location
  535. return None
  536. def gen_setup(self, filename, fragment, tmpdir):
  537. match = EGG_FRAGMENT.match(fragment)
  538. dists = match and [
  539. d for d in
  540. interpret_distro_name(filename, match.group(1), None) if d.version
  541. ] or []
  542. if len(dists)==1: # unambiguous ``#egg`` fragment
  543. basename = os.path.basename(filename)
  544. # Make sure the file has been downloaded to the temp dir.
  545. if os.path.dirname(filename) != tmpdir:
  546. dst = os.path.join(tmpdir, basename)
  547. from setuptools.command.easy_install import samefile
  548. if not samefile(filename, dst):
  549. shutil.copy2(filename, dst)
  550. filename=dst
  551. with open(os.path.join(tmpdir, 'setup.py'), 'w') as file:
  552. file.write(
  553. "from setuptools import setup\n"
  554. "setup(name=%r, version=%r, py_modules=[%r])\n"
  555. % (
  556. dists[0].project_name, dists[0].version,
  557. os.path.splitext(basename)[0]
  558. )
  559. )
  560. return filename
  561. elif match:
  562. raise DistutilsError(
  563. "Can't unambiguously interpret project/version identifier %r; "
  564. "any dashes in the name or version should be escaped using "
  565. "underscores. %r" % (fragment,dists)
  566. )
  567. else:
  568. raise DistutilsError(
  569. "Can't process plain .py files without an '#egg=name-version'"
  570. " suffix to enable automatic setup script generation."
  571. )
  572. dl_blocksize = 8192
  573. def _download_to(self, url, filename):
  574. self.info("Downloading %s", url)
  575. # Download the file
  576. fp, info = None, None
  577. try:
  578. checker = HashChecker.from_url(url)
  579. fp = self.open_url(strip_fragment(url))
  580. if isinstance(fp, HTTPError):
  581. raise DistutilsError(
  582. "Can't download %s: %s %s" % (url, fp.code,fp.msg)
  583. )
  584. headers = fp.info()
  585. blocknum = 0
  586. bs = self.dl_blocksize
  587. size = -1
  588. if "content-length" in headers:
  589. # Some servers return multiple Content-Length headers :(
  590. sizes = get_all_headers(headers, 'Content-Length')
  591. size = max(map(int, sizes))
  592. self.reporthook(url, filename, blocknum, bs, size)
  593. with open(filename,'wb') as tfp:
  594. while True:
  595. block = fp.read(bs)
  596. if block:
  597. checker.feed(block)
  598. tfp.write(block)
  599. blocknum += 1
  600. self.reporthook(url, filename, blocknum, bs, size)
  601. else:
  602. break
  603. self.check_hash(checker, filename, tfp)
  604. return headers
  605. finally:
  606. if fp: fp.close()
  607. def reporthook(self, url, filename, blocknum, blksize, size):
  608. pass # no-op
  609. def open_url(self, url, warning=None):
  610. if url.startswith('file:'):
  611. return local_open(url)
  612. try:
  613. return open_with_auth(url, self.opener)
  614. except (ValueError, httplib.InvalidURL) as v:
  615. msg = ' '.join([str(arg) for arg in v.args])
  616. if warning:
  617. self.warn(warning, msg)
  618. else:
  619. raise DistutilsError('%s %s' % (url, msg))
  620. except urllib2.HTTPError as v:
  621. return v
  622. except urllib2.URLError as v:
  623. if warning:
  624. self.warn(warning, v.reason)
  625. else:
  626. raise DistutilsError("Download error for %s: %s"
  627. % (url, v.reason))
  628. except httplib.BadStatusLine as v:
  629. if warning:
  630. self.warn(warning, v.line)
  631. else:
  632. raise DistutilsError(
  633. '%s returned a bad status line. The server might be '
  634. 'down, %s' %
  635. (url, v.line)
  636. )
  637. except httplib.HTTPException as v:
  638. if warning:
  639. self.warn(warning, v)
  640. else:
  641. raise DistutilsError("Download error for %s: %s"
  642. % (url, v))
  643. def _download_url(self, scheme, url, tmpdir):
  644. # Determine download filename
  645. #
  646. name, fragment = egg_info_for_url(url)
  647. if name:
  648. while '..' in name:
  649. name = name.replace('..','.').replace('\\','_')
  650. else:
  651. name = "__downloaded__" # default if URL has no path contents
  652. if name.endswith('.egg.zip'):
  653. name = name[:-4] # strip the extra .zip before download
  654. filename = os.path.join(tmpdir,name)
  655. # Download the file
  656. #
  657. if scheme=='svn' or scheme.startswith('svn+'):
  658. return self._download_svn(url, filename)
  659. elif scheme=='git' or scheme.startswith('git+'):
  660. return self._download_git(url, filename)
  661. elif scheme.startswith('hg+'):
  662. return self._download_hg(url, filename)
  663. elif scheme=='file':
  664. return url2pathname(urlparse(url)[2])
  665. else:
  666. self.url_ok(url, True) # raises error if not allowed
  667. return self._attempt_download(url, filename)
  668. def scan_url(self, url):
  669. self.process_url(url, True)
  670. def _attempt_download(self, url, filename):
  671. headers = self._download_to(url, filename)
  672. if 'html' in headers.get('content-type','').lower():
  673. return self._download_html(url, headers, filename)
  674. else:
  675. return filename
  676. def _download_html(self, url, headers, filename):
  677. file = open(filename)
  678. for line in file:
  679. if line.strip():
  680. # Check for a subversion index page
  681. if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
  682. # it's a subversion index page:
  683. file.close()
  684. os.unlink(filename)
  685. return self._download_svn(url, filename)
  686. break # not an index page
  687. file.close()
  688. os.unlink(filename)
  689. raise DistutilsError("Unexpected HTML page found at "+url)
  690. def _download_svn(self, url, filename):
  691. url = url.split('#',1)[0] # remove any fragment for svn's sake
  692. creds = ''
  693. if url.lower().startswith('svn:') and '@' in url:
  694. scheme, netloc, path, p, q, f = urlparse(url)
  695. if not netloc and path.startswith('//') and '/' in path[2:]:
  696. netloc, path = path[2:].split('/',1)
  697. auth, host = splituser(netloc)
  698. if auth:
  699. if ':' in auth:
  700. user, pw = auth.split(':',1)
  701. creds = " --username=%s --password=%s" % (user, pw)
  702. else:
  703. creds = " --username="+auth
  704. netloc = host
  705. url = urlunparse((scheme, netloc, url, p, q, f))
  706. self.info("Doing subversion checkout from %s to %s", url, filename)
  707. os.system("svn checkout%s -q %s %s" % (creds, url, filename))
  708. return filename
  709. @staticmethod
  710. def _vcs_split_rev_from_url(url, pop_prefix=False):
  711. scheme, netloc, path, query, frag = urlsplit(url)
  712. scheme = scheme.split('+', 1)[-1]
  713. # Some fragment identification fails
  714. path = path.split('#',1)[0]
  715. rev = None
  716. if '@' in path:
  717. path, rev = path.rsplit('@', 1)
  718. # Also, discard fragment
  719. url = urlunsplit((scheme, netloc, path, query, ''))
  720. return url, rev
  721. def _download_git(self, url, filename):
  722. filename = filename.split('#',1)[0]
  723. url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
  724. self.info("Doing git clone from %s to %s", url, filename)
  725. os.system("git clone --quiet %s %s" % (url, filename))
  726. if rev is not None:
  727. self.info("Checking out %s", rev)
  728. os.system("(cd %s && git checkout --quiet %s)" % (
  729. filename,
  730. rev,
  731. ))
  732. return filename
  733. def _download_hg(self, url, filename):
  734. filename = filename.split('#',1)[0]
  735. url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)
  736. self.info("Doing hg clone from %s to %s", url, filename)
  737. os.system("hg clone --quiet %s %s" % (url, filename))
  738. if rev is not None:
  739. self.info("Updating to %s", rev)
  740. os.system("(cd %s && hg up -C -r %s >&-)" % (
  741. filename,
  742. rev,
  743. ))
  744. return filename
  745. def debug(self, msg, *args):
  746. log.debug(msg, *args)
  747. def info(self, msg, *args):
  748. log.info(msg, *args)
  749. def warn(self, msg, *args):
  750. log.warn(msg, *args)
  751. # This pattern matches a character entity reference (a decimal numeric
  752. # references, a hexadecimal numeric reference, or a named reference).
  753. entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub
  754. def uchr(c):
  755. if not isinstance(c, int):
  756. return c
  757. if c>255: return unichr(c)
  758. return chr(c)
  759. def decode_entity(match):
  760. what = match.group(1)
  761. if what.startswith('#x'):
  762. what = int(what[2:], 16)
  763. elif what.startswith('#'):
  764. what = int(what[1:])
  765. else:
  766. what = name2codepoint.get(what, match.group(0))
  767. return uchr(what)
  768. def htmldecode(text):
  769. """Decode HTML entities in the given text."""
  770. return entity_sub(decode_entity, text)
  771. def socket_timeout(timeout=15):
  772. def _socket_timeout(func):
  773. def _socket_timeout(*args, **kwargs):
  774. old_timeout = socket.getdefaulttimeout()
  775. socket.setdefaulttimeout(timeout)
  776. try:
  777. return func(*args, **kwargs)
  778. finally:
  779. socket.setdefaulttimeout(old_timeout)
  780. return _socket_timeout
  781. return _socket_timeout
  782. def _encode_auth(auth):
  783. """
  784. A function compatible with Python 2.3-3.3 that will encode
  785. auth from a URL suitable for an HTTP header.
  786. >>> str(_encode_auth('username%3Apassword'))
  787. 'dXNlcm5hbWU6cGFzc3dvcmQ='
  788. Long auth strings should not cause a newline to be inserted.
  789. >>> long_auth = 'username:' + 'password'*10
  790. >>> chr(10) in str(_encode_auth(long_auth))
  791. False
  792. """
  793. auth_s = unquote(auth)
  794. # convert to bytes
  795. auth_bytes = auth_s.encode()
  796. # use the legacy interface for Python 2.3 support
  797. encoded_bytes = base64.encodestring(auth_bytes)
  798. # convert back to a string
  799. encoded = encoded_bytes.decode()
  800. # strip the trailing carriage return
  801. return encoded.replace('\n','')
  802. class Credential(object):
  803. """
  804. A username/password pair. Use like a namedtuple.
  805. """
  806. def __init__(self, username, password):
  807. self.username = username
  808. self.password = password
  809. def __iter__(self):
  810. yield self.username
  811. yield self.password
  812. def __str__(self):
  813. return '%(username)s:%(password)s' % vars(self)
  814. class PyPIConfig(ConfigParser.ConfigParser):
  815. def __init__(self):
  816. """
  817. Load from ~/.pypirc
  818. """
  819. defaults = dict.fromkeys(['username', 'password', 'repository'], '')
  820. ConfigParser.ConfigParser.__init__(self, defaults)
  821. rc = os.path.join(os.path.expanduser('~'), '.pypirc')
  822. if os.path.exists(rc):
  823. self.read(rc)
  824. @property
  825. def creds_by_repository(self):
  826. sections_with_repositories = [
  827. section for section in self.sections()
  828. if self.get(section, 'repository').strip()
  829. ]
  830. return dict(map(self._get_repo_cred, sections_with_repositories))
  831. def _get_repo_cred(self, section):
  832. repo = self.get(section, 'repository').strip()
  833. return repo, Credential(
  834. self.get(section, 'username').strip(),
  835. self.get(section, 'password').strip(),
  836. )
  837. def find_credential(self, url):
  838. """
  839. If the URL indicated appears to be a repository defined in this
  840. config, return the credential for that repository.
  841. """
  842. for repository, cred in self.creds_by_repository.items():
  843. if url.startswith(repository):
  844. return cred
  845. def open_with_auth(url, opener=urllib2.urlopen):
  846. """Open a urllib2 request, handling HTTP authentication"""
  847. scheme, netloc, path, params, query, frag = urlparse(url)
  848. # Double scheme does not raise on Mac OS X as revealed by a
  849. # failing test. We would expect "nonnumeric port". Refs #20.
  850. if netloc.endswith(':'):
  851. raise httplib.InvalidURL("nonnumeric port: ''")
  852. if scheme in ('http', 'https'):
  853. auth, host = splituser(netloc)
  854. else:
  855. auth = None
  856. if not auth:
  857. cred = PyPIConfig().find_credential(url)
  858. if cred:
  859. auth = str(cred)
  860. info = cred.username, url
  861. log.info('Authenticating as %s for %s (from .pypirc)' % info)
  862. if auth:
  863. auth = "Basic " + _encode_auth(auth)
  864. new_url = urlunparse((scheme,host,path,params,query,frag))
  865. request = urllib2.Request(new_url)
  866. request.add_header("Authorization", auth)
  867. else:
  868. request = urllib2.Request(url)
  869. request.add_header('User-Agent', user_agent)
  870. fp = opener(request)
  871. if auth:
  872. # Put authentication info back into request URL if same host,
  873. # so that links found on the page will work
  874. s2, h2, path2, param2, query2, frag2 = urlparse(fp.url)
  875. if s2==scheme and h2==host:
  876. fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2))
  877. return fp
  878. # adding a timeout to avoid freezing package_index
  879. open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)
  880. def fix_sf_url(url):
  881. return url # backward compatibility
  882. def local_open(url):
  883. """Read a local path, with special support for directories"""
  884. scheme, server, path, param, query, frag = urlparse(url)
  885. filename = url2pathname(path)
  886. if os.path.isfile(filename):
  887. return urllib2.urlopen(url)
  888. elif path.endswith('/') and os.path.isdir(filename):
  889. files = []
  890. for f in os.listdir(filename):
  891. if f=='index.html':
  892. with open(os.path.join(filename,f),'r') as fp:
  893. body = fp.read()
  894. break
  895. elif os.path.isdir(os.path.join(filename,f)):
  896. f+='/'
  897. files.append("<a href=%r>%s</a>" % (f,f))
  898. else:
  899. body = ("<html><head><title>%s</title>" % url) + \
  900. "</head><body>%s</body></html>" % '\n'.join(files)
  901. status, message = 200, "OK"
  902. else:
  903. status, message, body = 404, "Path not found", "Not found"
  904. headers = {'content-type': 'text/html'}
  905. return HTTPError(url, status, message, headers, StringIO(body))