encoding.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. import codecs
  2. import datetime
  3. import locale
  4. from decimal import Decimal
  5. from urllib.parse import quote
  6. from django.utils.functional import Promise
  7. class DjangoUnicodeDecodeError(UnicodeDecodeError):
  8. def __init__(self, obj, *args):
  9. self.obj = obj
  10. super().__init__(*args)
  11. def __str__(self):
  12. return "%s. You passed in %r (%s)" % (
  13. super().__str__(),
  14. self.obj,
  15. type(self.obj),
  16. )
  17. def smart_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  18. """
  19. Return a string representing 's'. Treat bytestrings using the 'encoding'
  20. codec.
  21. If strings_only is True, don't convert (some) non-string-like objects.
  22. """
  23. if isinstance(s, Promise):
  24. # The input is the result of a gettext_lazy() call.
  25. return s
  26. return force_str(s, encoding, strings_only, errors)
  27. _PROTECTED_TYPES = (
  28. type(None),
  29. int,
  30. float,
  31. Decimal,
  32. datetime.datetime,
  33. datetime.date,
  34. datetime.time,
  35. )
  36. def is_protected_type(obj):
  37. """Determine if the object instance is of a protected type.
  38. Objects of protected types are preserved as-is when passed to
  39. force_str(strings_only=True).
  40. """
  41. return isinstance(obj, _PROTECTED_TYPES)
  42. def force_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  43. """
  44. Similar to smart_str(), except that lazy instances are resolved to
  45. strings, rather than kept as lazy objects.
  46. If strings_only is True, don't convert (some) non-string-like objects.
  47. """
  48. # Handle the common case first for performance reasons.
  49. if issubclass(type(s), str):
  50. return s
  51. if strings_only and is_protected_type(s):
  52. return s
  53. try:
  54. if isinstance(s, bytes):
  55. s = str(s, encoding, errors)
  56. else:
  57. s = str(s)
  58. except UnicodeDecodeError as e:
  59. raise DjangoUnicodeDecodeError(s, *e.args)
  60. return s
  61. def smart_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  62. """
  63. Return a bytestring version of 's', encoded as specified in 'encoding'.
  64. If strings_only is True, don't convert (some) non-string-like objects.
  65. """
  66. if isinstance(s, Promise):
  67. # The input is the result of a gettext_lazy() call.
  68. return s
  69. return force_bytes(s, encoding, strings_only, errors)
  70. def force_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  71. """
  72. Similar to smart_bytes, except that lazy instances are resolved to
  73. strings, rather than kept as lazy objects.
  74. If strings_only is True, don't convert (some) non-string-like objects.
  75. """
  76. # Handle the common case first for performance reasons.
  77. if isinstance(s, bytes):
  78. if encoding == "utf-8":
  79. return s
  80. else:
  81. return s.decode("utf-8", errors).encode(encoding, errors)
  82. if strings_only and is_protected_type(s):
  83. return s
  84. if isinstance(s, memoryview):
  85. return bytes(s)
  86. return str(s).encode(encoding, errors)
  87. def iri_to_uri(iri):
  88. """
  89. Convert an Internationalized Resource Identifier (IRI) portion to a URI
  90. portion that is suitable for inclusion in a URL.
  91. This is the algorithm from RFC 3987 Section 3.1, slightly simplified since
  92. the input is assumed to be a string rather than an arbitrary byte stream.
  93. Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or
  94. b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded
  95. result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/').
  96. """
  97. # The list of safe characters here is constructed from the "reserved" and
  98. # "unreserved" characters specified in RFC 3986 Sections 2.2 and 2.3:
  99. # reserved = gen-delims / sub-delims
  100. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  101. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  102. # / "*" / "+" / "," / ";" / "="
  103. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  104. # Of the unreserved characters, urllib.parse.quote() already considers all
  105. # but the ~ safe.
  106. # The % character is also added to the list of safe characters here, as the
  107. # end of RFC 3987 Section 3.1 specifically mentions that % must not be
  108. # converted.
  109. if iri is None:
  110. return iri
  111. elif isinstance(iri, Promise):
  112. iri = str(iri)
  113. return quote(iri, safe="/#%[]=:;$&()+,!?*@'~")
  114. # List of byte values that uri_to_iri() decodes from percent encoding.
  115. # First, the unreserved characters from RFC 3986:
  116. _ascii_ranges = [[45, 46, 95, 126], range(65, 91), range(97, 123)]
  117. _hextobyte = {
  118. (fmt % char).encode(): bytes((char,))
  119. for ascii_range in _ascii_ranges
  120. for char in ascii_range
  121. for fmt in ["%02x", "%02X"]
  122. }
  123. # And then everything above 128, because bytes ≥ 128 are part of multibyte
  124. # Unicode characters.
  125. _hexdig = "0123456789ABCDEFabcdef"
  126. _hextobyte.update(
  127. {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig[8:] for b in _hexdig}
  128. )
  129. def uri_to_iri(uri):
  130. """
  131. Convert a Uniform Resource Identifier(URI) into an Internationalized
  132. Resource Identifier(IRI).
  133. This is the algorithm from RFC 3987 Section 3.2, excluding step 4.
  134. Take an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and return
  135. a string containing the encoded result (e.g. '/I%20♥%20Django/').
  136. """
  137. if uri is None:
  138. return uri
  139. uri = force_bytes(uri)
  140. # Fast selective unquote: First, split on '%' and then starting with the
  141. # second block, decode the first 2 bytes if they represent a hex code to
  142. # decode. The rest of the block is the part after '%AB', not containing
  143. # any '%'. Add that to the output without further processing.
  144. bits = uri.split(b"%")
  145. if len(bits) == 1:
  146. iri = uri
  147. else:
  148. parts = [bits[0]]
  149. append = parts.append
  150. hextobyte = _hextobyte
  151. for item in bits[1:]:
  152. hex = item[:2]
  153. if hex in hextobyte:
  154. append(hextobyte[item[:2]])
  155. append(item[2:])
  156. else:
  157. append(b"%")
  158. append(item)
  159. iri = b"".join(parts)
  160. return repercent_broken_unicode(iri).decode()
  161. def escape_uri_path(path):
  162. """
  163. Escape the unsafe characters from the path portion of a Uniform Resource
  164. Identifier (URI).
  165. """
  166. # These are the "reserved" and "unreserved" characters specified in RFC
  167. # 3986 Sections 2.2 and 2.3:
  168. # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
  169. # unreserved = alphanum | mark
  170. # mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
  171. # The list of safe characters here is constructed subtracting ";", "=",
  172. # and "?" according to RFC 3986 Section 3.3.
  173. # The reason for not subtracting and escaping "/" is that we are escaping
  174. # the entire path, not a path segment.
  175. return quote(path, safe="/:@&+$,-_.!~*'()")
  176. def punycode(domain):
  177. """Return the Punycode of the given domain if it's non-ASCII."""
  178. return domain.encode("idna").decode("ascii")
  179. def repercent_broken_unicode(path):
  180. """
  181. As per RFC 3987 Section 3.2, step three of converting a URI into an IRI,
  182. repercent-encode any octet produced that is not part of a strictly legal
  183. UTF-8 octet sequence.
  184. """
  185. changed_parts = []
  186. while True:
  187. try:
  188. path.decode()
  189. except UnicodeDecodeError as e:
  190. # CVE-2019-14235: A recursion shouldn't be used since the exception
  191. # handling uses massive amounts of memory
  192. repercent = quote(path[e.start : e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
  193. changed_parts.append(path[: e.start] + repercent.encode())
  194. path = path[e.end :]
  195. else:
  196. return b"".join(changed_parts) + path
  197. def filepath_to_uri(path):
  198. """Convert a file system path to a URI portion that is suitable for
  199. inclusion in a URL.
  200. Encode certain chars that would normally be recognized as special chars
  201. for URIs. Do not encode the ' character, as it is a valid character
  202. within URIs. See the encodeURIComponent() JavaScript function for details.
  203. """
  204. if path is None:
  205. return path
  206. # I know about `os.sep` and `os.altsep` but I want to leave
  207. # some flexibility for hardcoding separators.
  208. return quote(str(path).replace("\\", "/"), safe="/~!*()'")
  209. def get_system_encoding():
  210. """
  211. The encoding for the character type functions. Fallback to 'ascii' if the
  212. #encoding is unsupported by Python or could not be determined. See tickets
  213. #10335 and #5846.
  214. """
  215. try:
  216. encoding = locale.getlocale()[1] or "ascii"
  217. codecs.lookup(encoding)
  218. except Exception:
  219. encoding = "ascii"
  220. return encoding
  221. DEFAULT_LOCALE_ENCODING = get_system_encoding()