dateparser.languages.dictionary — DateParser 1.4.2 documentation

# Source code for dateparser.languages.dictionary

    from itertools import chain, zip_longest
    from operator import methodcaller

    import regex as re

    from dateparser.timezone_parser import is_timezone_token
    from dateparser.utils import normalize_unicode

    PARSER_HARDCODED_TOKENS = [":", ".", " ", "-", "/"]
    PARSER_KNOWN_TOKENS = ["am", "pm", "UTC", "GMT", "Z"]
    ALWAYS_KEEP_TOKENS = ["+"] + PARSER_HARDCODED_TOKENS
    KNOWN_WORD_TOKENS = [
        "monday",
        "tuesday",
        "wednesday",
        "thursday",
        "friday",
        "saturday",
        "sunday",
        "january",
        "february",
        "march",
        "april",
        "may",
        "june",
        "july",
        "august",
        "september",
        "october",
        "november",
        "december",
        "decade",
        "year",
        "month",
        "week",
        "day",
        "hour",
        "minute",
        "second",
        "ago",
        "in",
        "am",
        "pm",
    ]

    PARENTHESES_PATTERN = re.compile(r"[\(\)]")
    NUMERAL_PATTERN = re.compile(r"(\d+)")
    KEEP_TOKEN_PATTERN = re.compile(r"^.*[^\W_].*$", flags=re.U)

    class UnknownTokenError(Exception):
        pass

    def _parse_bool(value):
        if isinstance(value, bool):
            return value
        if isinstance(value, str):
            return value.strip().lower() in ("true", "1", "yes")
        return False

    class Dictionary:
        """
        Class that modifies and stores translations and handles splitting of date string.

        :param locale_info:
            Locale info (translation data) of the locale.
        :type language_info: dict

        :param settings:
            Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
        :type settings: dict

        :return: a Dictionary instance.
        """

        _split_regex_cache = {}
        _sorted_words_cache = {}
        _split_relative_regex_cache = {}
        _sorted_relative_strings_cache = {}
        _match_relative_regex_cache = {}

        def __init__(self, locale_info, settings=None):
            dictionary = {}
            self._settings = settings
            self.info = locale_info

            if "skip" in locale_info:
                skip = map(methodcaller("lower"), locale_info["skip"])
                dictionary.update(zip_longest(skip, [], fillvalue=None))
            if "pertain" in locale_info:
                pertain = map(methodcaller("lower"), locale_info["pertain"])
                dictionary.update(zip_longest(pertain, [], fillvalue=None))
            for word in KNOWN_WORD_TOKENS:
                if word in locale_info:
                    translations = map(methodcaller("lower"), locale_info[word])
                    dictionary.update(zip_longest(translations, [], fillvalue=word))
            dictionary.update(zip_longest(ALWAYS_KEEP_TOKENS, ALWAYS_KEEP_TOKENS))
            dictionary.update(
                zip_longest(
                    map(methodcaller("lower"), PARSER_KNOWN_TOKENS), PARSER_KNOWN_TOKENS
                )
            )

            relative_type = locale_info.get("relative-type", {})
            for key, value in relative_type.items():
                relative_translations = map(methodcaller("lower"), value)
                dictionary.update(zip_longest(relative_translations, [], fillvalue=key))

            self._dictionary = dictionary

            no_word_spacing = locale_info.get("no_word_spacing", False)
            self._no_word_spacing = _parse_bool(no_word_spacing)

            relative_type_regex = locale_info.get("relative-type-regex", {})
            self._relative_strings = list(chain.from_iterable(relative_type_regex.values()))

        def __contains__(self, key):
            if key in self._settings.SKIP_TOKENS:
                return True
            return self._dictionary.__contains__(key)

        def __getitem__(self, key):
            if key in self._settings.SKIP_TOKENS:
                return None
            return self._dictionary.__getitem__(key)

        def __iter__(self):
            return chain(self._settings.SKIP_TOKENS, iter(self._dictionary))

        def are_tokens_valid(self, tokens):
            """
            Check if tokens are valid tokens for the locale.

            :param tokens:
                a list of string tokens.
            :type tokens: list

            :return: True if tokens are valid, False otherwise.
            """
            has_only_keep_tokens = not set(tokens) - set(ALWAYS_KEEP_TOKENS)
            if has_only_keep_tokens:
                return False
            match_relative_regex = self._get_match_relative_regex_cache()
            return all(
                self._is_known_token(token, match_relative_regex) for token in tokens
            )

        def _is_known_token(self, token, match_relative_regex):
            """Whether ``token`` is recognised by this locale: a number, a relative
            expression, or a dictionary word (the same per-token check that
            :meth:`are_tokens_valid` applies)."""
            return bool(
                token.isdigit() or match_relative_regex.match(token) or token in self
            )

        def _strip_unknown_edge_tokens(self, tokens):
            """Return ``tokens`` without the leading and trailing tokens that this
            locale does not recognise (neither digits, relative expressions, nor
            dictionary words), along with any whitespace those removals leave at
            the edges. Interior tokens are left untouched. Used when the
            ``IGNORE_SURROUNDING_TEXT`` setting is enabled.

            A timezone abbreviation that is the final token (e.g. a trailing
            ``"EST"``) is kept even though the locale dictionary does not list it,
            so that the offset is applied rather than silently discarded. It
            survives translation and is popped later by
            :func:`~dateparser.timezone_parser.pop_tz_offset_from_string`. The
            timezone is kept only when it is the last token: if further
            unrecognized text follows it, the tokenizer merges the two and the
            timezone is dropped together with that text, like any other edge noise.

            This timezone exception is deliberately made at the trailing edge
            only: a timezone conventionally follows the date it qualifies
            (``"... 1:21 PM EST"``) and does not precede it, so the leading edge
            keeps stripping unrecognized tokens unconditionally."""
            match_relative_regex = self._get_match_relative_regex_cache()

            def is_extra_text(token):
                return token.isspace() or not self._is_known_token(
                    token, match_relative_regex
                )

            start, end = 0, len(tokens)
            # Leading edge: drop every unrecognized token. A timezone does not
            # precede the date it qualifies, so no timezone exception is made here.
            while start < end and is_extra_text(tokens[start]):
                start += 1
            # Trailing edge: drop unrecognized tokens, but keep a recognized
            # timezone abbreviation (e.g. ``" est"``) so its offset is applied.
            while (
                end > start
                and is_extra_text(tokens[end - 1])
                and not is_timezone_token(tokens[end - 1])
            ):
                end -= 1
            return tokens[start:end]

        def split(self, string, keep_formatting=False):
            """
            Split the date string using translations in locale info.

            :param string:
                Date string to be splitted.
            :type string:
                str

            :param keep_formatting:
                If True, retain formatting of the date string.
            :type keep_formatting: bool

            :return: A list of string tokens formed after splitting the date string.
            """
            if not string:
                return string

            split_relative_regex = self._get_split_relative_regex_cache()
            match_relative_regex = self._get_match_relative_regex_cache()

            tokens = split_relative_regex.split(string)

            for i, token in enumerate(tokens):
                if match_relative_regex.match(token):
                    tokens[i] = [token]
                    continue
                tokens[i] = self._split_by_known_words(token, keep_formatting)

            return list(filter(bool, chain.from_iterable(tokens)))

        def _add_to_cache(self, value, cache):
            cache.setdefault(self._settings.registry_key, {})[self.info["name"]] = value
            if (
                self._settings.CACHE_SIZE_LIMIT
                and len(cache) > self._settings.CACHE_SIZE_LIMIT
            ):
                cache.pop(list(cache.keys())[0])

        def _split_by_known_words(self, string: str, keep_formatting: bool):
            regex = self._get_split_regex_cache()
            splitted = []
            unknown = string

            while unknown:
                match = regex.match(string)

                if not match:
                    curr_split = (
                        self._split_by_numerals(string, keep_formatting)
                        if self._should_capture(string, keep_formatting)
                        else []
                    )
                    unknown = ""
                else:
                    unparsed, known, unknown = match.groups()
                    curr_split = (
                        [known] if self._should_capture(known, keep_formatting) else []
                    )

                    if unparsed and self._should_capture(unparsed, keep_formatting):
                        curr_split = (
                            self._split_by_numerals(unparsed, keep_formatting) + curr_split
                        )

                    if unknown:
                        string = unknown if string != unknown else ""

                for token in curr_split:
                    if (
                        splitted
                        and splitted[-1].isdigit()
                        and token in {"st", "nd", "rd", "th"}
                    ):
                        continue

                    splitted.append(token)

            return splitted

        def _split_by_numerals(self, string, keep_formatting):
            return [
                token
                for token in NUMERAL_PATTERN.split(string)
                if self._should_capture(token, keep_formatting)
            ]

        def _should_capture(self, token, keep_formatting):
            return (
                keep_formatting
                or token in ALWAYS_KEEP_TOKENS
                or KEEP_TOKEN_PATTERN.match(token)
            )

        def _get_sorted_words_from_cache(self):
            if (
                self._settings.registry_key not in self._sorted_words_cache
                or self.info["name"]
                not in self._sorted_words_cache[self._settings.registry_key]
            ):
                self._add_to_cache(
                    cache=self._sorted_words_cache,
                    value=sorted([key for key in self], key=len, reverse=True),
                )
            return self._sorted_words_cache[self._settings.registry_key][self.info["name"]]

        def _get_split_regex_cache(self):
            if (
                self._settings.registry_key not in self._split_regex_cache
                or self.info["name"]
                not in self._split_regex_cache[self._settings.registry_key]
            ):
                self._construct_split_regex()
            return self._split_regex_cache[self._settings.registry_key][self.info["name"]]

        def _construct_split_regex(self):
            known_words_group = "|".join(
                map(re.escape, self._get_sorted_words_from_cache())
            )
            if self._no_word_spacing:
                regex = r"^(.*?)({})(.*)$".format(known_words_group)
            else:
                regex = r"^(.*?(?:\A|\W|_|\d))({})((?:\Z|\W|_|\d).*)$".format(
                    known_words_group
                )
            self._add_to_cache(
                cache=self._split_regex_cache,
                value=re.compile(regex, re.UNICODE | re.IGNORECASE),
            )

        def _get_sorted_relative_strings_from_cache(self):
            if (
                self._settings.registry_key not in self._sorted_relative_strings_cache
                or self.info["name"]
                not in self._sorted_relative_strings_cache[self._settings.registry_key]
            ):
                self._add_to_cache(
                    cache=self._sorted_relative_strings_cache,
                    value=sorted(
                        [
                            PARENTHESES_PATTERN.sub("", key)
                            for key in self._relative_strings
                        ],
                        key=len,
                        reverse=True,
                    ),
                )
            return self._sorted_relative_strings_cache[self._settings.registry_key][
                self.info["name"]
            ]

        def _get_split_relative_regex_cache(self):
            if (
                self._settings.registry_key not in self._split_relative_regex_cache
                or self.info["name"]
                not in self._split_relative_regex_cache[self._settings.registry_key]
            ):
                self._construct_split_relative_regex()
            return self._split_relative_regex_cache[self._settings.registry_key][
                self.info["name"]
            ]

        def _construct_split_relative_regex(self):
            known_relative_strings_group = "|".join(
                self._get_sorted_relative_strings_from_cache()
            )
            if self._no_word_spacing:
                regex = "({})".format(known_relative_strings_group)
            else:
                regex = "(?<=(?:\\A|\\W|_))({})(?=(?:\\Z|\\W|_))".format(
                    known_relative_strings_group
                )
            self._add_to_cache(
                cache=self._split_relative_regex_cache,
                value=re.compile(regex, re.UNICODE | re.IGNORECASE),
            )

        def _get_match_relative_regex_cache(self):
            if (
                self._settings.registry_key not in self._match_relative_regex_cache
                or self.info["name"]
                not in self._match_relative_regex_cache[self._settings.registry_key]
            ):
                self._construct_match_relative_regex()
            return self._match_relative_regex_cache[self._settings.registry_key][
                self.info["name"]
            ]

        def _construct_match_relative_regex(self):
            known_relative_strings_group = "|".join(
                self._get_sorted_relative_strings_from_cache()
            )
            regex = "^({})$".format(known_relative_strings_group)
            self._add_to_cache(
                cache=self._match_relative_regex_cache,
                value=re.compile(regex, re.UNICODE | re.IGNORECASE),
            )

    class NormalizedDictionary(Dictionary):
        def __init__(self, locale_info, settings=None):
            super().__init__(locale_info, settings)
            self._normalize()

        def _normalize(self):
            new_dict = {}
            conflicting_keys = []
            for key, value in self._dictionary.items():
                normalized = normalize_unicode(key)
                if key != normalized and normalized in self._dictionary:
                    conflicting_keys.append(key)
                else:
                    new_dict[normalized] = value
            for key in conflicting_keys:
                normalized = normalize_unicode(key)
                if key in (self.info.get("skip", []) + self.info.get("pertain", [])):
                    new_dict[normalized] = self._dictionary[key]
            self._dictionary = new_dict
            self._relative_strings = list(map(normalize_unicode, self._relative_strings))