| import email.feedparser |
| import email.header |
| import email.message |
| import email.parser |
| import email.policy |
| import sys |
| import typing |
| from typing import Dict, List, Optional, Tuple, Union, cast |
| |
| if sys.version_info >= (3, 8): # pragma: no cover |
| from typing import TypedDict |
| else: # pragma: no cover |
| if typing.TYPE_CHECKING: |
| from typing_extensions import TypedDict |
| else: |
| try: |
| from typing_extensions import TypedDict |
| except ImportError: |
| |
| class TypedDict: |
| def __init_subclass__(*_args, **_kwargs): |
| pass |
| |
| |
| # The RawMetadata class attempts to make as few assumptions about the underlying |
| # serialization formats as possible. The idea is that as long as a serialization |
| # formats offer some very basic primitives in *some* way then we can support |
| # serializing to and from that format. |
| class RawMetadata(TypedDict, total=False): |
| """A dictionary of raw core metadata. |
| |
| Each field in core metadata maps to a key of this dictionary (when data is |
| provided). The key is lower-case and underscores are used instead of dashes |
| compared to the equivalent core metadata field. Any core metadata field that |
| can be specified multiple times or can hold multiple values in a single |
| field have a key with a plural name. |
| |
| Core metadata fields that can be specified multiple times are stored as a |
| list or dict depending on which is appropriate for the field. Any fields |
| which hold multiple values in a single field are stored as a list. |
| |
| """ |
| |
| # Metadata 1.0 - PEP 241 |
| metadata_version: str |
| name: str |
| version: str |
| platforms: List[str] |
| summary: str |
| description: str |
| keywords: List[str] |
| home_page: str |
| author: str |
| author_email: str |
| license: str |
| |
| # Metadata 1.1 - PEP 314 |
| supported_platforms: List[str] |
| download_url: str |
| classifiers: List[str] |
| requires: List[str] |
| provides: List[str] |
| obsoletes: List[str] |
| |
| # Metadata 1.2 - PEP 345 |
| maintainer: str |
| maintainer_email: str |
| requires_dist: List[str] |
| provides_dist: List[str] |
| obsoletes_dist: List[str] |
| requires_python: str |
| requires_external: List[str] |
| project_urls: Dict[str, str] |
| |
| # Metadata 2.0 |
| # PEP 426 attempted to completely revamp the metadata format |
| # but got stuck without ever being able to build consensus on |
| # it and ultimately ended up withdrawn. |
| # |
| # However, a number of tools had started emiting METADATA with |
| # `2.0` Metadata-Version, so for historical reasons, this version |
| # was skipped. |
| |
| # Metadata 2.1 - PEP 566 |
| description_content_type: str |
| provides_extra: List[str] |
| |
| # Metadata 2.2 - PEP 643 |
| dynamic: List[str] |
| |
| # Metadata 2.3 - PEP 685 |
| # No new fields were added in PEP 685, just some edge case were |
| # tightened up to provide better interoptability. |
| |
| |
| _STRING_FIELDS = { |
| "author", |
| "author_email", |
| "description", |
| "description_content_type", |
| "download_url", |
| "home_page", |
| "license", |
| "maintainer", |
| "maintainer_email", |
| "metadata_version", |
| "name", |
| "requires_python", |
| "summary", |
| "version", |
| } |
| |
| _LIST_STRING_FIELDS = { |
| "classifiers", |
| "dynamic", |
| "obsoletes", |
| "obsoletes_dist", |
| "platforms", |
| "provides", |
| "provides_dist", |
| "provides_extra", |
| "requires", |
| "requires_dist", |
| "requires_external", |
| "supported_platforms", |
| } |
| |
| |
| def _parse_keywords(data: str) -> List[str]: |
| """Split a string of comma-separate keyboards into a list of keywords.""" |
| return [k.strip() for k in data.split(",")] |
| |
| |
| def _parse_project_urls(data: List[str]) -> Dict[str, str]: |
| """Parse a list of label/URL string pairings separated by a comma.""" |
| urls = {} |
| for pair in data: |
| # Our logic is slightly tricky here as we want to try and do |
| # *something* reasonable with malformed data. |
| # |
| # The main thing that we have to worry about, is data that does |
| # not have a ',' at all to split the label from the Value. There |
| # isn't a singular right answer here, and we will fail validation |
| # later on (if the caller is validating) so it doesn't *really* |
| # matter, but since the missing value has to be an empty str |
| # and our return value is dict[str, str], if we let the key |
| # be the missing value, then they'd have multiple '' values that |
| # overwrite each other in a accumulating dict. |
| # |
| # The other potentional issue is that it's possible to have the |
| # same label multiple times in the metadata, with no solid "right" |
| # answer with what to do in that case. As such, we'll do the only |
| # thing we can, which is treat the field as unparseable and add it |
| # to our list of unparsed fields. |
| parts = [p.strip() for p in pair.split(",", 1)] |
| parts.extend([""] * (max(0, 2 - len(parts)))) # Ensure 2 items |
| |
| # TODO: The spec doesn't say anything about if the keys should be |
| # considered case sensitive or not... logically they should |
| # be case-preserving and case-insensitive, but doing that |
| # would open up more cases where we might have duplicate |
| # entries. |
| label, url = parts |
| if label in urls: |
| # The label already exists in our set of urls, so this field |
| # is unparseable, and we can just add the whole thing to our |
| # unparseable data and stop processing it. |
| raise KeyError("duplicate labels in project urls") |
| urls[label] = url |
| |
| return urls |
| |
| |
| def _get_payload(msg: email.message.Message, source: Union[bytes, str]) -> str: |
| """Get the body of the message.""" |
| # If our source is a str, then our caller has managed encodings for us, |
| # and we don't need to deal with it. |
| if isinstance(source, str): |
| payload: str = msg.get_payload() |
| return payload |
| # If our source is a bytes, then we're managing the encoding and we need |
| # to deal with it. |
| else: |
| bpayload: bytes = msg.get_payload(decode=True) |
| try: |
| return bpayload.decode("utf8", "strict") |
| except UnicodeDecodeError: |
| raise ValueError("payload in an invalid encoding") |
| |
| |
| # The various parse_FORMAT functions here are intended to be as lenient as |
| # possible in their parsing, while still returning a correctly typed |
| # RawMetadata. |
| # |
| # To aid in this, we also generally want to do as little touching of the |
| # data as possible, except where there are possibly some historic holdovers |
| # that make valid data awkward to work with. |
| # |
| # While this is a lower level, intermediate format than our ``Metadata`` |
| # class, some light touch ups can make a massive difference in usability. |
| |
| # Map METADATA fields to RawMetadata. |
| _EMAIL_TO_RAW_MAPPING = { |
| "author": "author", |
| "author-email": "author_email", |
| "classifier": "classifiers", |
| "description": "description", |
| "description-content-type": "description_content_type", |
| "download-url": "download_url", |
| "dynamic": "dynamic", |
| "home-page": "home_page", |
| "keywords": "keywords", |
| "license": "license", |
| "maintainer": "maintainer", |
| "maintainer-email": "maintainer_email", |
| "metadata-version": "metadata_version", |
| "name": "name", |
| "obsoletes": "obsoletes", |
| "obsoletes-dist": "obsoletes_dist", |
| "platform": "platforms", |
| "project-url": "project_urls", |
| "provides": "provides", |
| "provides-dist": "provides_dist", |
| "provides-extra": "provides_extra", |
| "requires": "requires", |
| "requires-dist": "requires_dist", |
| "requires-external": "requires_external", |
| "requires-python": "requires_python", |
| "summary": "summary", |
| "supported-platform": "supported_platforms", |
| "version": "version", |
| } |
| |
| |
| def parse_email(data: Union[bytes, str]) -> Tuple[RawMetadata, Dict[str, List[str]]]: |
| """Parse a distribution's metadata. |
| |
| This function returns a two-item tuple of dicts. The first dict is of |
| recognized fields from the core metadata specification. Fields that can be |
| parsed and translated into Python's built-in types are converted |
| appropriately. All other fields are left as-is. Fields that are allowed to |
| appear multiple times are stored as lists. |
| |
| The second dict contains all other fields from the metadata. This includes |
| any unrecognized fields. It also includes any fields which are expected to |
| be parsed into a built-in type but were not formatted appropriately. Finally, |
| any fields that are expected to appear only once but are repeated are |
| included in this dict. |
| |
| """ |
| raw: Dict[str, Union[str, List[str], Dict[str, str]]] = {} |
| unparsed: Dict[str, List[str]] = {} |
| |
| if isinstance(data, str): |
| parsed = email.parser.Parser(policy=email.policy.compat32).parsestr(data) |
| else: |
| parsed = email.parser.BytesParser(policy=email.policy.compat32).parsebytes(data) |
| |
| # We have to wrap parsed.keys() in a set, because in the case of multiple |
| # values for a key (a list), the key will appear multiple times in the |
| # list of keys, but we're avoiding that by using get_all(). |
| for name in frozenset(parsed.keys()): |
| # Header names in RFC are case insensitive, so we'll normalize to all |
| # lower case to make comparisons easier. |
| name = name.lower() |
| |
| # We use get_all() here, even for fields that aren't multiple use, |
| # because otherwise someone could have e.g. two Name fields, and we |
| # would just silently ignore it rather than doing something about it. |
| headers = parsed.get_all(name) |
| |
| # The way the email module works when parsing bytes is that it |
| # unconditionally decodes the bytes as ascii using the surrogateescape |
| # handler. When you pull that data back out (such as with get_all() ), |
| # it looks to see if the str has any surrogate escapes, and if it does |
| # it wraps it in a Header object instead of returning the string. |
| # |
| # As such, we'll look for those Header objects, and fix up the encoding. |
| value = [] |
| # Flag if we have run into any issues processing the headers, thus |
| # signalling that the data belongs in 'unparsed'. |
| valid_encoding = True |
| for h in headers: |
| # It's unclear if this can return more types than just a Header or |
| # a str, so we'll just assert here to make sure. |
| assert isinstance(h, (email.header.Header, str)) |
| |
| # If it's a header object, we need to do our little dance to get |
| # the real data out of it. In cases where there is invalid data |
| # we're going to end up with mojibake, but there's no obvious, good |
| # way around that without reimplementing parts of the Header object |
| # ourselves. |
| # |
| # That should be fine since, if mojibacked happens, this key is |
| # going into the unparsed dict anyways. |
| if isinstance(h, email.header.Header): |
| # The Header object stores it's data as chunks, and each chunk |
| # can be independently encoded, so we'll need to check each |
| # of them. |
| chunks: List[Tuple[bytes, Optional[str]]] = [] |
| for bin, encoding in email.header.decode_header(h): |
| try: |
| bin.decode("utf8", "strict") |
| except UnicodeDecodeError: |
| # Enable mojibake. |
| encoding = "latin1" |
| valid_encoding = False |
| else: |
| encoding = "utf8" |
| chunks.append((bin, encoding)) |
| |
| # Turn our chunks back into a Header object, then let that |
| # Header object do the right thing to turn them into a |
| # string for us. |
| value.append(str(email.header.make_header(chunks))) |
| # This is already a string, so just add it. |
| else: |
| value.append(h) |
| |
| # We've processed all of our values to get them into a list of str, |
| # but we may have mojibake data, in which case this is an unparsed |
| # field. |
| if not valid_encoding: |
| unparsed[name] = value |
| continue |
| |
| raw_name = _EMAIL_TO_RAW_MAPPING.get(name) |
| if raw_name is None: |
| # This is a bit of a weird situation, we've encountered a key that |
| # we don't know what it means, so we don't know whether it's meant |
| # to be a list or not. |
| # |
| # Since we can't really tell one way or another, we'll just leave it |
| # as a list, even though it may be a single item list, because that's |
| # what makes the most sense for email headers. |
| unparsed[name] = value |
| continue |
| |
| # If this is one of our string fields, then we'll check to see if our |
| # value is a list of a single item. If it is then we'll assume that |
| # it was emitted as a single string, and unwrap the str from inside |
| # the list. |
| # |
| # If it's any other kind of data, then we haven't the faintest clue |
| # what we should parse it as, and we have to just add it to our list |
| # of unparsed stuff. |
| if raw_name in _STRING_FIELDS and len(value) == 1: |
| raw[raw_name] = value[0] |
| # If this is one of our list of string fields, then we can just assign |
| # the value, since email *only* has strings, and our get_all() call |
| # above ensures that this is a list. |
| elif raw_name in _LIST_STRING_FIELDS: |
| raw[raw_name] = value |
| # Special Case: Keywords |
| # The keywords field is implemented in the metadata spec as a str, |
| # but it conceptually is a list of strings, and is serialized using |
| # ", ".join(keywords), so we'll do some light data massaging to turn |
| # this into what it logically is. |
| elif raw_name == "keywords" and len(value) == 1: |
| raw[raw_name] = _parse_keywords(value[0]) |
| # Special Case: Project-URL |
| # The project urls is implemented in the metadata spec as a list of |
| # specially-formatted strings that represent a key and a value, which |
| # is fundamentally a mapping, however the email format doesn't support |
| # mappings in a sane way, so it was crammed into a list of strings |
| # instead. |
| # |
| # We will do a little light data massaging to turn this into a map as |
| # it logically should be. |
| elif raw_name == "project_urls": |
| try: |
| raw[raw_name] = _parse_project_urls(value) |
| except KeyError: |
| unparsed[name] = value |
| # Nothing that we've done has managed to parse this, so it'll just |
| # throw it in our unparseable data and move on. |
| else: |
| unparsed[name] = value |
| |
| # We need to support getting the Description from the message payload in |
| # addition to getting it from the the headers. This does mean, though, there |
| # is the possibility of it being set both ways, in which case we put both |
| # in 'unparsed' since we don't know which is right. |
| try: |
| payload = _get_payload(parsed, data) |
| except ValueError: |
| unparsed.setdefault("description", []).append( |
| parsed.get_payload(decode=isinstance(data, bytes)) |
| ) |
| else: |
| if payload: |
| # Check to see if we've already got a description, if so then both |
| # it, and this body move to unparseable. |
| if "description" in raw: |
| description_header = cast(str, raw.pop("description")) |
| unparsed.setdefault("description", []).extend( |
| [description_header, payload] |
| ) |
| elif "description" in unparsed: |
| unparsed["description"].append(payload) |
| else: |
| raw["description"] = payload |
| |
| # We need to cast our `raw` to a metadata, because a TypedDict only support |
| # literal key names, but we're computing our key names on purpose, but the |
| # way this function is implemented, our `TypedDict` can only have valid key |
| # names. |
| return cast(RawMetadata, raw), unparsed |
