beancount.parser

def book(incomplete_entries, options_map, initial_balances=None):
    """Book inventory lots and complete all positions with incomplete numbers.

    Args:
      incomplete_entries: A list of directives, with some postings possibly left
        with incomplete amounts as produced by the parser.
      options_map: An options dict as produced by the parser.
      initial_balances: A dict of (account, inventory) pairs to start booking from.
        This is useful when attempting to book on top of an existing state.
    Returns:
      A pair of
        entries: A list of completed entries with all their postings completed.
        errors: New errors produced during interpolation.
    """
    # Get the list of booking methods for each account.
    booking_methods = collections.defaultdict(lambda: options_map["booking_method"])
    for entry in incomplete_entries:
        if isinstance(entry, data.Open) and entry.booking:
            booking_methods[entry.account] = entry.booking

    # Do the booking here!
    entries, booking_errors = booking_full.book(incomplete_entries, options_map,
                                                booking_methods, initial_balances)

    # Check for MISSING elements remaining.
    missing_errors = validate_missing_eliminated(entries, options_map)

    return entries, (booking_errors + missing_errors)

def convert_lot_specs_to_lots(entries):
    """For all the entries, convert the posting's position's CostSpec to Cost
    instances. In the simple method, the data provided in the CostSpec must
    unambiguously provide a way to compute the cost amount.

    This essentially replicates the way the old parser used to work, but
    allowing positions to have the fuzzy lot specifications instead of the
    resolved ones. We used to simply compute the costs locally, and this gets
    rid of the CostSpec to produce the Cost without fuzzy matching. This is only
    there for the sake of transition to the new matching logic.

    Args:
      entries: A list of incomplete directives as per the parser.
    Returns:
      A list of entries whose postings's position costs have been converted to
      Cost instances but that may still be incomplete.
    Raises:
      ValueError: If there's a unacceptable number.
    """
    new_entries = []
    errors = []
    for entry in entries:
        if not isinstance(entry, data.Transaction):
            new_entries.append(entry)
            continue

        new_postings = []
        for posting in entry.postings:
            try:
                units = posting.units
                cost_spec = posting.cost
                cost = convert_spec_to_cost(units, cost_spec)
                if cost_spec is not None and cost is None:
                    errors.append(
                        BookingError(entry.meta,
                                     "Cost syntax not supported; cost spec ignored",
                                     None))

                if cost and isinstance(units, amount.Amount):
                    # If there is a cost, we don't allow either a cost value of
                    # zero, nor a zero number of units. Note that we allow a price
                    # of zero as the only special case (for conversion entries), but
                    # never for costs.
                    if units.number == ZERO:
                        raise ValueError('Amount is zero: "{}"'.format(units))
                    if cost.number is not None and cost.number < ZERO:
                        raise ValueError('Cost is negative: "{}"'.format(cost))
            except ValueError as exc:
                errors.append(BookingError(entry.meta, str(exc), None))
                cost = None
            new_postings.append(posting._replace(cost=cost))
        new_entries.append(entry._replace(postings=new_postings))
    return new_entries, errors

def convert_spec_to_cost(units, cost_spec):
    """Convert a posting's CostSpec instance to a Cost.

    Args:
      units: An instance of Amount.
      cost_spec: An instance of CostSpec.
    Returns:
      An instance of Cost.
    """
    cost = cost_spec
    errors = []
    if isinstance(units, amount.Amount):
        currency = units.currency
        if cost_spec is not None:
            number_per, number_total, cost_currency, date, label, merge = cost_spec

            # Compute the cost.
            if number_per is not MISSING or number_total is not None:
                if number_total is not None:
                    # Compute the per-unit cost if there is some total cost
                    # component involved.
                    units_num = units.number
                    cost_total = number_total
                    if number_per is not MISSING:
                        cost_total += number_per * units_num
                    unit_cost = cost_total / abs(units_num)
                else:
                    unit_cost = number_per
                cost = position.Cost(unit_cost, cost_currency, date, label)
            else:
                cost = None
    return cost

def validate_inventory_booking(entries, unused_options_map, booking_methods):
    """Validate that no position at cost is allowed to go negative.

    This routine checks that when a posting reduces a position, existing or not,
    that the subsequent inventory does not result in a position with a negative
    number of units. A negative number of units would only be required for short
    trades of trading spreads on futures, and right now this is not supported.
    It would not be difficult to support this, however, but we want to be strict
    about it, because being pedantic about this is otherwise a great way to
    detect user data entry mistakes.

    Args:
      entries: A list of directives.
      unused_options_map: An options map.
      booking_methods: A mapping of account name to booking method, accumulated
        in the main loop.
    Returns:
      A list of errors.

    """
    errors = []
    balances = collections.defaultdict(inventory.Inventory)
    for entry in entries:
        if isinstance(entry, data.Transaction):
            for posting in entry.postings:
                # Update the balance of each posting on its respective account
                # without allowing booking to a negative position, and if an error
                # is encountered, catch it and return it.
                running_balance = balances[posting.account]
                position_, _ = running_balance.add_position(posting)

                # Skip this check if the booking method is set to ignore it.
                if booking_methods.get(posting.account, None) == data.Booking.NONE:
                    continue

                # Check if the resulting inventory is mixed, which is not
                # allowed under the STRICT method.
                if running_balance.is_mixed():
                    errors.append(
                        BookingError(
                            entry.meta,
                            ("Reducing position results in inventory with positive "
                             "and negative lots: {}").format(position_),
                            entry))

    return errors

def validate_missing_eliminated(entries, unused_options_map):
    """Validate that all the missing bits of postings have been eliminated.

    Args:
      entries: A list of directives.
      unused_options_map: An options map.
    Returns:
      A list of errors.
    """
    errors = []
    for entry in entries:
        if isinstance(entry, data.Transaction):
            for posting in entry.postings:
                units = posting.units
                cost = posting.cost
                if (MISSING in (units.number, units.currency) or
                    cost is not None and MISSING in (cost.number, cost.currency,
                                                     cost.date, cost.label)):
                    errors.append(
                        BookingError(entry.meta,
                                     "Transaction has incomplete elements",
                                     entry))
                    break
    return errors

def book(entries, options_map, methods, initial_balances=None):
    """Interpolate missing data from the entries using the full historical algorithm.
    See the internal implementation _book() for details.
    This method only stripes some of the return values.

    See _book() for arguments and return values.
    """
    entries, errors, _ = _book(entries, options_map, methods, initial_balances)
    return entries, errors

def book_reductions(entry, group_postings, balances,
                    methods):
    """Book inventory reductions against the ante-balances.

    This function accepts a dict of (account, Inventory balance) and for each
    posting that is a reduction against its inventory, attempts to find a
    corresponding lot or list of lots to reduce the balance with.

    * For reducing lots, the CostSpec instance of the posting is replaced by a
      Cost instance.

    * For augmenting lots, the CostSpec instance of the posting is left alone,
      except for its date, which is inherited from the parent Transaction.

    Args:
      entry: An instance of Transaction. This is only used to refer to when
        logging errors.
      group_postings: A list of Posting instances for the group.
      balances: A dict of account name to inventory contents.
      methods: A mapping of account name to their corresponding booking
        method enum.
    Returns:
      A pair of
        booked_postings: A list of booked postings, with reducing lots resolved
          against specific position in the corresponding accounts'
          ante-inventory balances. Note single reducing posting in the input may
          result in multiple postings in the output. Also note that augmenting
          postings held-at-cost will still refer to 'cost' instances of
          CostSpec, left to be interpolated later.
        errors: A list of errors, if there were any.
    """
    errors = []

    # A local copy of the balances dictionary which is updated just for the
    # duration of this function's updates, in order to take into account the
    # cumulative effect of all the postings inferred here
    local_balances = {}

    empty = inventory.Inventory()
    booked_postings = []
    for posting in group_postings:
        # Process a single posting.
        units = posting.units
        costspec = posting.cost
        account = posting.account

        # Note: We ensure there is no mutation on 'balances' to keep this
        # function without side-effects. Note that we may be able to optimize
        # performance later on by giving up this property.
        #
        # Also note that if there is no existing balance, then won't be any lot
        # reduction because none of the postings will be able to match against
        # any currencies of the balance.
        if account not in local_balances:
            previous_balance = balances.get(account, empty)
            local_balances[account] = copy.copy(previous_balance)
        balance = local_balances[account]

        # Check if this is a lot held at cost.
        if costspec is None or units.number is MISSING:
            # This posting is not held at cost; we do nothing.
            booked_postings.append(posting)
        else:
            # This posting is held at cost; figure out if it's a reduction or an
            # augmentation.
            method = methods[account]
            if (method is not Booking.NONE and
                balance is not None and
                balance.is_reduced_by(units)):
                # This posting is a reduction.

                # Match the positions.
                cost_number = compute_cost_number(costspec, units)
                matches = []
                for position in balance:
                    # Skip inventory contents of a different currency.
                    if (units.currency and
                        position.units.currency != units.currency):
                        continue
                    # Skip balance positions not held at cost.
                    if position.cost is None:
                        continue
                    if (cost_number is not None and
                        position.cost.number != cost_number):
                        continue
                    if (isinstance(costspec.currency, str) and
                        position.cost.currency != costspec.currency):
                        continue
                    if (costspec.date and
                        position.cost.date != costspec.date):
                        continue
                    if (costspec.label and
                        position.cost.label != costspec.label):
                        continue
                    matches.append(position)

                # Check for ambiguous matches.
                if len(matches) == 0:
                    errors.append(
                        ReductionError(entry.meta,
                                       'No position matches "{}" against balance {}'.format(
                                           posting, balance),
                                       entry))
                    return [], errors  # This is irreconcilable, remove these postings.

                # TODO(blais): We'll have to change this, as we want to allow
                # positions crossing from negative to positive and vice-versa in
                # a simple application. See {d3cbd78f1029}.
                reduction_postings, matched_postings, ambi_errors = (
                    booking_method.handle_ambiguous_matches(entry, posting, matches,
                                                            method))
                if ambi_errors:
                    errors.extend(ambi_errors)
                    return [], errors

                # Add the reductions to the resulting list of booked postings.
                booked_postings.extend(reduction_postings)

                # Update the local balance in order to avoid matching against
                # the same postings twice when processing multiple postings in
                # the same transaction. Note that we only do this for postings
                # held at cost because the other postings may need interpolation
                # in order to be resolved properly.
                for posting in reduction_postings:
                    balance.add_position(posting)
            else:
                # This posting is an augmentation.
                #
                # Note that we do not convert the CostSpec instances to Cost
                # instances, because we want to let the subsequent interpolation
                # process able to interpolate either the cost per-unit or the
                # total cost, separately.

                # Put in the date of the parent Transaction if there is no
                # explicit date specified on the spec.
                if costspec.date is None:
                    dated_costspec = costspec._replace(date=entry.date)
                    posting = posting._replace(cost=dated_costspec)

                # FIXME: Insert unique ids for trade tracking; right now this
                # creates ambiguous matches errors (and it shouldn't).
                # # Insert a unique label if there isn't one.
                # if posting.cost is not None and posting.cost.label is None:
                #     posting = posting._replace(
                #         cost=posting.cost._replace(label=unique_label()))

                booked_postings.append(posting)

    return booked_postings, errors

def categorize_by_currency(entry, balances):
    """Group the postings by the currency they declare.

    This is used to prepare the postings for the next stages: Interpolation and
    booking will then be carried out separately on each currency group. At the
    outset of this routine, we should have distinct groups of currencies without
    any ambiguities regarding which currency they need to balance against.

    Here's how this works.

    - First we apply the constraint that cost-currency and price-currency must
      match, if there is both a cost and a price. This reduces the space of
      possibilities somewhat.

    - If the currency is explicitly specified, we put the posting in that
      currency's bucket.

    - If not, we have a few methods left to disambiguate the currency:

      1. We look at the remaining postings... if they are all of a single
         currency, the posting must be in that currency too.

      2. If we cannot do that, we inspect the contents of the inventory of the
         account for the posting. If all the contents are of a single currency,
         we use that one.

    Args:
      postings: A list of incomplete postings to categorize.
      balances: A dict of currency to inventory contents before the transaction is
        applied.
    Returns:
      A list of (currency string, list of tuples) items describing each postings
      and its interpolated currencies, and a list of generated errors for
      currency interpolation. The entry's original postings are left unmodified.
      Each tuple in the value-list contains:
        index: The posting index in the original entry.
        units_currency: The interpolated currency for units.
        cost_currency: The interpolated currency for cost.
        price_currency: The interpolated currency for price.
    """
    errors = []

    groups = collections.defaultdict(list)
    sortdict = {}
    auto_postings = []
    unknown = []
    for index, posting in enumerate(entry.postings):
        units = posting.units
        cost = posting.cost
        price = posting.price

        # Extract and override the currencies locally.
        units_currency = (units.currency
                          if units is not MISSING and units is not None
                          else None)
        cost_currency = (cost.currency
                         if cost is not MISSING and cost is not None
                         else None)
        price_currency = (price.currency
                          if price is not MISSING and price is not None
                          else None)

        # First we apply the constraint that cost-currency and price-currency
        # must match, if there is both a cost and a price. This reduces the
        # space of possibilities somewhat.
        if cost_currency is MISSING and isinstance(price_currency, str):
            cost_currency = price_currency
        if price_currency is MISSING and isinstance(cost_currency, str):
            price_currency = cost_currency

        refer = Refer(index, units_currency, cost_currency, price_currency)

        if units is MISSING and price_currency is None:
            # Bucket auto-postings separately from unknown.
            auto_postings.append(refer)
        else:
            # Bucket with what we know so far.
            currency = get_bucket_currency(refer)
            if currency is not None:
                sortdict.setdefault(currency, index)
                groups[currency].append(refer)
            else:
                # If we need to infer the currency, store in unknown.
                unknown.append(refer)

    # We look at the remaining postings... if they are all of a single currency,
    # the posting must be in that currency too.
    if unknown and len(unknown) == 1 and len(groups) == 1:
        (index, units_currency, cost_currency, price_currency) = unknown.pop()

        other_currency = next(iter(groups.keys()))
        if price_currency is None and cost_currency is None:
            # Infer to the units currency.
            units_currency = other_currency
        else:
            # Infer to the cost and price currencies.
            if price_currency is MISSING:
                price_currency = other_currency
            if cost_currency is MISSING:
                cost_currency = other_currency

        refer = Refer(index, units_currency, cost_currency, price_currency)
        currency = get_bucket_currency(refer)
        assert currency is not None
        sortdict.setdefault(currency, index)
        groups[currency].append(refer)

    # Finally, try to resolve all the unknown legs using the inventory contents
    # of each account.
    for refer in unknown:
        (index, units_currency, cost_currency, price_currency) = refer
        posting = entry.postings[index]
        balance = balances.get(posting.account, None)
        if balance is None:
            balance = inventory.Inventory()

        if units_currency is MISSING:
            balance_currencies = balance.currencies()
            if len(balance_currencies) == 1:
                units_currency = balance_currencies.pop()

        if cost_currency is MISSING or price_currency is MISSING:
            balance_cost_currencies = balance.cost_currencies()
            if len(balance_cost_currencies) == 1:
                balance_cost_currency = balance_cost_currencies.pop()
                if price_currency is MISSING:
                    price_currency = balance_cost_currency
                if cost_currency is MISSING:
                    cost_currency = balance_cost_currency

        refer = Refer(index, units_currency, cost_currency, price_currency)
        currency = get_bucket_currency(refer)
        if currency is not None:
            sortdict.setdefault(currency, index)
            groups[currency].append(refer)
        else:
            errors.append(
                CategorizationError(posting.meta,
                                    "Failed to categorize posting {}".format(index + 1),
                                    entry))

    # Fill in missing units currencies if some remain as missing. This may occur
    # if we used the cost or price to bucket the currency but the units currency
    # was missing.
    for currency, refers in groups.items():
        for rindex, refer in enumerate(refers):
            if refer.units_currency is MISSING:
                posting = entry.postings[refer.index]
                balance = balances.get(posting.account, None)
                if balance is None:
                    continue
                balance_currencies = balance.currencies()
                if len(balance_currencies) == 1:
                    refers[rindex] = refer._replace(units_currency=balance_currencies.pop())

    # Deal with auto-postings.
    if len(auto_postings) > 1:
        refer = auto_postings[-1]
        posting = entry.postings[refer.index]
        errors.append(
            CategorizationError(posting.meta,
                                "You may not have more than one auto-posting per currency",
                                entry))
        auto_postings = auto_postings[0:1]
    for refer in auto_postings:
        for currency, glist in groups.items():
            sortdict.setdefault(currency, refer.index)
            glist.append(Refer(refer.index, currency, None, None))

    # Issue error for all currencies which we could not resolve.
    for currency, refers in groups.items():
        for refer in refers:
            posting = entry.postings[refer.index]
            for currency, name in [(refer.units_currency, 'units'),
                                   (refer.cost_currency, 'cost'),
                                   (refer.price_currency, 'price')]:
                if currency is MISSING:
                    errors.append(CategorizationError(
                        posting.meta,
                        "Could not resolve {} currency".format(name),
                        entry))

    sorted_groups = sorted(groups.items(), key=lambda item: sortdict[item[0]])
    return sorted_groups, errors

def compute_cost_number(costspec, units):
    """Given a CostSpec, return the cost number, if possible to compute.

    Args:
      costspec: A parsed instance of CostSpec.
      units: An instance of Amount for the units of the position.
    Returns:
      If it is not possible to calculate the cost, return None.
      Otherwise, returns a Decimal instance, the per-unit cost.
    """
    number_per = costspec.number_per
    number_total = costspec.number_total
    if MISSING in (number_per, number_total):
        return None
    if number_total is not None:
        # Compute the per-unit cost if there is some total cost
        # component involved.
        cost_total = number_total
        units_number = abs(units.number)
        if number_per is not None:
            cost_total += number_per * units_number
        unit_cost = cost_total / units_number
    elif number_per is None:
        return None
    else:
        unit_cost = number_per
    return unit_cost

def convert_costspec_to_cost(posting):
    """Convert an instance of CostSpec to Cost, if present on the posting.

    If the posting has no cost, it itself is just returned.

    Args:
      posting: An instance of Posting.
    Returns:
      An instance of Posting with a possibly replaced 'cost' attribute.
    """
    cost = posting.cost
    if isinstance(cost, position.CostSpec):
        if cost is not None:
            number_per = cost.number_per
            number_total = cost.number_total
            if number_total is not None:
                # Compute the per-unit cost if there is some total cost
                # component involved.
                units_number = abs(posting.units.number)
                cost_total = number_total
                if number_per is not MISSING:
                    cost_total += number_per * units_number
                unit_cost = cost_total / units_number
            else:
                unit_cost = number_per
            new_cost = Cost(unit_cost, cost.currency, cost.date, cost.label)
            posting = posting._replace(units=posting.units, cost=new_cost)
    return posting

def get_bucket_currency(refer):
    """Given currency references for a posting, return the bucket currency.

    Args:
      refer: An instance of Refer.
    Returns:
      A currency string.
    """
    currency = None
    if isinstance(refer.cost_currency, str):
        currency = refer.cost_currency
    elif isinstance(refer.price_currency, str):
        currency = refer.price_currency
    elif (refer.cost_currency is None and
          refer.price_currency is None and
          isinstance(refer.units_currency, str)):
        currency = refer.units_currency
    return currency

def has_self_reduction(postings, methods):
    """Return true if the postings potentially reduce each other at cost.

    Args:
      postings: A list of postings with uninterpolated CostSpec cost instances.
      methods: A mapping of account name to their corresponding booking
        method.
    Returns:
      A boolean, true if there's a potential for self-reduction.
    """
    # A mapping of (currency, cost-currency) and sign.
    cost_changes = {}
    for posting in postings:
        cost = posting.cost
        if cost is None:
            continue
        if methods[posting.account] is Booking.NONE:
            continue
        key = (posting.account, posting.units.currency)
        sign = 1 if posting.units.number > ZERO else -1
        if cost_changes.setdefault(key, sign) != sign:
            return True
    return False

def interpolate_group(postings, balances, currency, tolerances):
    """Interpolate missing numbers in the set of postings.

    Args:
      postings: A list of Posting instances.
      balances: A dict of account to its ante-inventory.
      currency: The weight currency of this group, used for reporting errors.
      tolerances: A dict of currency to tolerance values.
    Returns:
      A tuple of
        postings: A list of new posting instances.
        errors: A list of errors generated during interpolation.
        interpolated: A boolean, true if we did have to interpolate.

      In the case of an error, this returns the original list of postings, which
      is still incomplete. If an error is returned, you should probably skip the
      transaction altogether, or just not include the postings in it. (An
      alternative behaviour would be to return only the list of valid postings,
      but that would likely result in an unbalanced transaction. We do it this
      way by choice.)
    """
    errors = []

    # Figure out which type of amount is missing, by creating a list of
    # incomplete postings and which type of units is missing.
    incomplete = []
    for index, posting in enumerate(postings):
        units = posting.units
        cost = posting.cost
        price = posting.price

        # Identify incomplete parts of the Posting components.
        if units.number is MISSING:
            incomplete.append((MissingType.UNITS, index))

        if isinstance(cost, CostSpec):
            if cost and cost.number_per is MISSING:
                incomplete.append((MissingType.COST_PER, index))
            if cost and cost.number_total is MISSING:
                incomplete.append((MissingType.COST_TOTAL, index))
        else:
            # Check that a resolved instance of Cost never needs interpolation.
            #
            # Note that in theory we could support the interpolation of regular
            # per-unit costs in these if we wanted to; but because they're all
            # reducing postings that have been booked earlier, those never need
            # to be interpolated.
            if cost is not None:
                assert isinstance(cost.number, Decimal), (
                    "Internal error: cost has no number: {}; on postings: {}".format(
                        cost, postings))

        if price and price.number is MISSING:
            incomplete.append((MissingType.PRICE, index))

    # The replacement posting for the incomplete posting of this group.
    new_posting = None

    if len(incomplete) == 0:
        # If there are no missing numbers, just convert the CostSpec to Cost and
        # return that.
        out_postings = [convert_costspec_to_cost(posting)
                        for posting in postings]

    elif len(incomplete) > 1:
        # If there is more than a single value to be interpolated, generate an
        # error and return no postings.
        _, posting_index = incomplete[0]
        errors.append(InterpolationError(
            postings[posting_index].meta,
            "Too many missing numbers for currency group '{}'".format(currency),
            None))
        out_postings = []

    else:
        # If there is a single missing number, calculate it and fill it in here.
        missing, index = incomplete[0]
        incomplete_posting = postings[index]

        # Convert augmenting postings' costs from CostSpec to corresponding Cost
        # instances, except for the incomplete posting.
        new_postings = [(posting
                         if posting is incomplete_posting
                         else convert_costspec_to_cost(posting))
                        for posting in postings]

        # Compute the balance of the other postings.
        residual = interpolate.compute_residual(posting
                                                for posting in new_postings
                                                if posting is not incomplete_posting)
        assert len(residual) < 2, "Internal error in grouping postings by currencies."
        if not residual.is_empty():
            respos = next(iter(residual))
            assert respos.cost is None, (
                "Internal error; cost appears in weight calculation.")
            assert respos.units.currency == currency, (
                "Internal error; residual different than currency group.")
            weight = -respos.units.number
            weight_currency = respos.units.currency
        else:
            weight = ZERO
            weight_currency = currency

        if missing == MissingType.UNITS:
            units = incomplete_posting.units
            cost = incomplete_posting.cost
            if cost:
                # Handle the special case where we only have total cost.
                if cost.number_per == ZERO:
                    errors.append(InterpolationError(
                        incomplete_posting.meta,
                        "Cannot infer per-unit cost only from total", None))
                    return postings, errors, True

                assert cost.currency == weight_currency, (
                    "Internal error; residual currency different than missing currency.")
                cost_total = cost.number_total or ZERO
                units_number = (weight - cost_total) / cost.number_per

            elif incomplete_posting.price:
                assert incomplete_posting.price.currency == weight_currency, (
                    "Internal error; residual currency different than missing currency.")
                units_number = weight / incomplete_posting.price.number

            else:
                assert units.currency == weight_currency, (
                    "Internal error; residual currency different than missing currency.")
                units_number = weight

            # Quantize the interpolated units if necessary.
            units_number = interpolate.quantize_with_tolerance(tolerances,
                                                               units.currency,
                                                               units_number)

            if weight != ZERO:
                new_pos = Position(Amount(units_number, units.currency), cost)
                new_posting = incomplete_posting._replace(units=new_pos.units,
                                                          cost=new_pos.cost)
            else:
                new_posting = None

        elif missing == MissingType.COST_PER:
            units = incomplete_posting.units
            cost = incomplete_posting.cost
            assert cost.currency == weight_currency, (
                "Internal error; residual currency different than missing currency.")
            if units.number != ZERO:
                number_per = (weight - (cost.number_total or ZERO)) / units.number
                new_cost = cost._replace(number_per=number_per)
                new_pos = Position(units, new_cost)
                new_posting = incomplete_posting._replace(units=new_pos.units,
                                                          cost=new_pos.cost)
            else:
                new_posting = None

        elif missing == MissingType.COST_TOTAL:
            units = incomplete_posting.units
            cost = incomplete_posting.cost
            assert cost.currency == weight_currency, (
                "Internal error; residual currency different than missing currency.")
            number_total = (weight - cost.number_per * units.number)
            new_cost = cost._replace(number_total=number_total)
            new_pos = Position(units, new_cost)
            new_posting = incomplete_posting._replace(units=new_pos.units,
                                                      cost=new_pos.cost)

        elif missing == MissingType.PRICE:
            units = incomplete_posting.units
            cost = incomplete_posting.cost
            if cost is not None:
                errors.append(InterpolationError(
                    incomplete_posting.meta,
                    "Cannot infer price for postings with units held at cost", None))
                return postings, errors, True
            else:
                price = incomplete_posting.price
                assert price.currency == weight_currency, (
                    "Internal error; residual currency different than missing currency.")
                new_price_number = abs(weight / units.number)
                new_posting = incomplete_posting._replace(price=Amount(new_price_number,
                                                                       price.currency))

        else:
            assert False, "Internal error; Invalid missing type."

        # Replace the number in the posting.
        if new_posting is not None:
            # Set meta-data on the new posting to indicate it was interpolated.
            if new_posting.meta is None:
                new_posting = new_posting._replace(meta={})
            new_posting.meta[interpolate.AUTOMATIC_META] = True

            # Convert augmenting posting costs from CostSpec to a corresponding
            # Cost instance.
            new_postings[index] = convert_costspec_to_cost(new_posting)
        else:
            del new_postings[index]
        out_postings = new_postings

    assert all(not isinstance(posting.cost, CostSpec)
               for posting in out_postings)

    # Check that units are non-zero and that no cost remains negative; issue an
    # error if this is the case.
    for posting in out_postings:
        if posting.cost is None:
            continue
        # If there is a cost, we don't allow either a cost value of zero,
        # nor a zero number of units. Note that we allow a price of zero as
        # the only special case allowed (for conversion entries), but never
        # for costs.
        if posting.units.number == ZERO:
            errors.append(InterpolationError(
                posting.meta,
                'Amount is zero: "{}"'.format(posting.units), None))
        if posting.cost.number is not None and posting.cost.number < ZERO:
            errors.append(InterpolationError(
                posting.meta,
                'Cost is negative: "{}"'.format(posting.cost), None))

    return out_postings, errors, (new_posting is not None)

def replace_currencies(postings, refer_groups):
    """Replace resolved currencies in the entry's Postings.

    This essentially applies the findings of categorize_by_currency() to produce
    new postings with all currencies resolved.

    Args:
      postings: A list of Posting instances to replace.
      refer_groups: A list of (currency, list of posting references) items as
        returned by categorize_by_currency().
    Returns:
      A new list of items of (currency, list of Postings), postings for which the
      currencies have been replaced by their interpolated currency values.
    """
    new_groups = []
    for currency, refers in refer_groups:
        new_postings = []
        for refer in sorted(refers, key=lambda r: r.index):
            posting = postings[refer.index]
            units = posting.units
            if units is MISSING or units is None:
                posting = posting._replace(units=Amount(MISSING, refer.units_currency))
            else:
                replace = False
                cost = posting.cost
                price = posting.price
                if units.currency is MISSING:
                    units = Amount(units.number, refer.units_currency)
                    replace = True
                if cost and cost.currency is MISSING:
                    cost = cost._replace(currency=refer.cost_currency)
                    replace = True
                if price and price.currency is MISSING:
                    price = Amount(price.number, refer.price_currency)
                    replace = True
                if replace:
                    posting = posting._replace(units=units, cost=cost, price=price)
            new_postings.append(posting)
        new_groups.append((currency, new_postings))
    return new_groups

def unique_label() -> str:
    "Return a globally unique label for cost entries."
    return str(uuid.uuid4())

def booking_method_AVERAGE(entry, posting, matches):
    """AVERAGE booking method implementation."""
    booked_reductions = []
    booked_matches = []
    errors = [AmbiguousMatchError(entry.meta, "AVERAGE method is not supported", entry)]
    return booked_reductions, booked_matches, errors, False

    # FIXME: Future implementation here.
    # pylint: disable=unreachable
    if False: # pylint: disable=using-constant-test
        # DISABLED - This is the code for AVERAGE, which is currently disabled.

        # If there is more than a single match we need to ultimately merge the
        # postings. Also, if the reducing posting provides a specific cost, we
        # need to update the cost basis as well. Both of these cases are carried
        # out by removing all the matches and readding them later on.
        if len(matches) == 1 and (
                not isinstance(posting.cost.number_per, Decimal) and
                not isinstance(posting.cost.number_total, Decimal)):
            # There is no cost. Just reduce the one leg. This should be the
            # normal case if we always merge augmentations and the user lets
            # Beancount deal with the cost.
            match = matches[0]
            sign = -1 if posting.units.number < ZERO else 1
            number = min(abs(match.units.number), abs(posting.units.number))
            match_units = Amount(number * sign, match.units.currency)
            booked_reductions.append(posting._replace(units=match_units, cost=match.cost))
            insufficient = (match_units.number != posting.units.number)
        else:
            # Merge the matching postings to a single one.
            merged_units = inventory.Inventory()
            merged_cost = inventory.Inventory()
            for match in matches:
                merged_units.add_amount(match.units)
                merged_cost.add_amount(convert.get_weight(match))
            if len(merged_units) != 1 or len(merged_cost) != 1:
                errors.append(
                    AmbiguousMatchError(
                        entry.meta,
                        'Cannot merge positions in multiple currencies: {}'.format(
                            ', '.join(position.to_string(match_posting)
                                      for match_posting in matches)), entry))
            else:
                if (isinstance(posting.cost.number_per, Decimal) or
                    isinstance(posting.cost.number_total, Decimal)):
                    errors.append(
                        AmbiguousMatchError(
                            entry.meta,
                            "Explicit cost reductions aren't supported yet: {}".format(
                                position.to_string(posting)), entry))
                else:
                    # Insert postings to remove all the matches.
                    booked_reductions.extend(
                        posting._replace(units=-match.units, cost=match.cost,
                                         flag=flags.FLAG_MERGING)
                        for match in matches)
                    units = merged_units[0].units
                    date = matches[0].cost.date  ## FIXME: Select which one,
                                                 ## oldest or latest.
                    cost_units = merged_cost[0].units
                    cost = Cost(cost_units.number/units.number, cost_units.currency,
                                date, None)

                    # Insert a posting to refill those with a replacement match.
                    booked_reductions.append(
                        posting._replace(units=units, cost=cost, flag=flags.FLAG_MERGING))

                    # Now, match the reducing request against this lot.
                    booked_reductions.append(
                        posting._replace(units=posting.units, cost=cost))
                    insufficient = abs(posting.units.number) > abs(units.number)

def booking_method_FIFO(entry, posting, matches):
    """FIFO booking method implementation."""
    return _booking_method_xifo(entry, posting, matches, "date", False)

def booking_method_HIFO(entry, posting, matches):
    """HIFO booking method implementation."""
    return _booking_method_xifo(entry, posting, matches, "number", True)

def booking_method_LIFO(entry, posting, matches):
    """LIFO booking method implementation."""
    return _booking_method_xifo(entry, posting, matches, "date", True)

def booking_method_NONE(entry, posting, matches):
    """NONE booking method implementation."""

    # This never needs to match against any existing positions... we
    # disregard the matches, there's never any error. Note that this never
    # gets called in practice, we want to treat NONE postings as
    # augmentations. Default behaviour is to return them with their original
    # CostSpec, and the augmentation code will handle signaling an error if
    # there is insufficient detail to carry out the conversion to an
    # instance of Cost.

    # Note that it's an interesting question whether a reduction on an
    # account with NONE method which happens to match a single position
    # ought to be matched against it. We don't allow it for now.

    return [posting], [], False

def booking_method_STRICT(entry, posting, matches):
    """Strict booking method. This method fails if there are ambiguous matches.
    """
    booked_reductions = []
    booked_matches = []
    errors = []
    insufficient = False

    # In strict mode, we require at most a single matching posting.
    if len(matches) > 1:
        # If the total requested to reduce matches the sum of all the
        # ambiguous postings, match against all of them.
        sum_matches = sum(p.units.number for p in matches)
        if sum_matches == -posting.units.number:
            booked_reductions.extend(
                posting._replace(units=-match.units, cost=match.cost)
                for match in matches)
        else:
            errors.append(
                AmbiguousMatchError(entry.meta,
                                    'Ambiguous matches for "{}": {}'.format(
                                        position.to_string(posting),
                                        ', '.join(position.to_string(match_posting)
                                                  for match_posting in matches)),
                                    entry))
    else:
        # Replace the posting's units and cost values.
        match = matches[0]
        sign = -1 if posting.units.number < ZERO else 1
        number = min(abs(match.units.number), abs(posting.units.number))
        match_units = Amount(number * sign, match.units.currency)
        booked_reductions.append(posting._replace(units=match_units, cost=match.cost))
        booked_matches.append(match)
        insufficient = (match_units.number != posting.units.number)

    return booked_reductions, booked_matches, errors, insufficient

def booking_method_STRICT_WITH_SIZE(entry, posting, matches):
    """Strict booking method, but disambiguate further with sizes.

    This booking method applies the same algorithm as the STRICT method, but if
    only one of the ambiguous lots matches the desired size, select that one
    automatically.
    """
    (booked_reductions, booked_matches, errors,
     insufficient) = booking_method_STRICT(entry, posting, matches)

    # If we couldn't match strictly, attempt to find a match with the same
    # number of units. If there is one or more of these, accept the oldest lot.
    if errors and len(matches) > 1:
        number = -posting.units.number
        matching_units = [match
                          for match in matches
                          if number == match.units.number]
        if matching_units:
            matching_units.sort(key=lambda match: match.cost.date)

            # Replace the posting's units and cost values.
            match = matching_units[0]
            booked_reductions.append(posting._replace(units=-match.units, cost=match.cost))
            booked_matches.append(match)
            insufficient = False
            errors = []

    return booked_reductions, booked_matches, errors, insufficient

def handle_ambiguous_matches(entry, posting, matches, method):
    """Handle ambiguous matches by dispatching to a particular method.

    Args:
      entry: The parent Transaction instance.
      posting: An instance of Posting, the reducing posting which we're
        attempting to match.
      matches: A list of matching Position instances from the ante-inventory.
        Those positions are known to already match the 'posting' spec.
      methods: A mapping of account name to their corresponding booking
        method.
    Returns:
      A triple of
        booked_reductions: A list of matched Posting instances, whose 'cost'
          attributes are ensured to be of type Cost.
        errors: A list of errors to be generated.
        insufficient: A boolean, true if we could not find enough matches to
        cover the entire position.
    """
    assert isinstance(method, Booking), (
        "Invalid type: {}".format(method))
    assert matches, "Internal error: Invalid call with no matches"

    #method = globals()['booking_method_{}'.format(method.name)]
    method = _BOOKING_METHODS[method]
    (booked_reductions,
     booked_matches, errors, insufficient) = method(entry, posting, matches)
    if insufficient:
        errors.append(
            AmbiguousMatchError(entry.meta,
                                'Not enough lots to reduce "{}": {}'.format(
                                    position.to_string(posting),
                                    ', '.join(position.to_string(match_posting)
                                              for match_posting in matches)),
                                entry))

    return booked_reductions, booked_matches, errors

def read_string_or_entries(entries_or_str, allow_incomplete=False):
    """Read a string of entries or just entries.

    Args:
      entries_or_str: Either a list of directives, or a string containing directives.
      allow_incomplete: A boolean, true if we allow incomplete inputs and perform
        light-weight booking.
    Returns:
      A list of directives.
    """
    if isinstance(entries_or_str, str):
        entries, errors, options_map = parser.parse_string(
            textwrap.dedent(entries_or_str))

        if allow_incomplete:
            # Do a simplistic local conversion in order to call the comparison.
            entries = [_local_booking(entry) for entry in entries]
        else:
            # Don't accept incomplete entries either.
            if any(parser.is_entry_incomplete(entry) for entry in entries):
                raise TestError("Entries in assertions may not use interpolation.")

            entries, booking_errors = booking.book(entries, options_map)
            errors = errors + booking_errors

        # Don't tolerate errors.
        if errors:
            oss = io.StringIO()
            printer.print_errors(errors, file=oss)
            raise TestError("Unexpected errors in expected: {}".format(oss.getvalue()))

    else:
        assert isinstance(entries_or_str, list), "Expecting list: {}".format(entries_or_str)
        entries = entries_or_str

    return entries

def render_entry_context(entries, options_map, entry, parsed_entry=None):
    """Render the context before and after a particular transaction is applied.

    Args:
      entries: A list of directives.
      options_map: A dict of options, as produced by the parser.
      entry: The entry instance which should be rendered. (Note that this object is
        expected to be in the set of entries, not just structurally equal.)
      parsed_entry: An optional incomplete, parsed but not booked nor interpolated
        entry. If this is provided, this is used for inspecting the list of prior
        accounts and it is also rendered.
    Returns:
      A multiline string of text, which consists of the context before the
      transaction is applied, the transaction itself, and the context after it
      is applied. You can just print that, it is in form that is intended to be
      consumed by the user.
    """
    oss = io.StringIO()
    pr = functools.partial(print, file=oss)
    header = "** {} --------------------------------"

    meta = entry.meta
    pr(header.format("Transaction Id"))
    pr()
    pr("Hash:{}".format(compare.hash_entry(entry)))
    pr("Location: {}:{}".format(meta["filename"], meta["lineno"]))
    pr()
    pr()

    # Get the list of accounts sorted by the order in which they appear in the
    # closest entry.
    order = {}
    if parsed_entry is None:
        parsed_entry = entry
    if isinstance(parsed_entry, data.Transaction):
        order = {posting.account: index
                 for index, posting in enumerate(parsed_entry.postings)}
    accounts = sorted(getters.get_entry_accounts(parsed_entry),
                      key=lambda account: order.get(account, 10000))

    # Accumulate the balances of these accounts up to the entry.
    balance_before, balance_after = interpolate.compute_entry_context(
        entries, entry, additional_accounts=accounts)

    # Create a format line for printing the contents of account balances.
    max_account_width = max(map(len, accounts)) if accounts else 1
    position_line = '{{:1}} {{:{width}}}  {{:>49}}'.format(width=max_account_width)

    # Print the context before.
    pr(header.format("Balances before transaction"))
    pr()
    before_hashes = set()
    average_costs = {}
    for account in accounts:
        balance = balance_before[account]

        pc_balances = balance.split()
        for currency, pc_balance in pc_balances.items():
            if len(pc_balance) > 1:
                average_costs[account] = pc_balance.average()

        positions = balance.get_positions()
        for position in positions:
            before_hashes.add((account, hash(position)))
            pr(position_line.format('', account, str(position)))
        if not positions:
            pr(position_line.format('', account, ''))
        pr()
    pr()

    # Print average cost per account, if relevant.
    if average_costs:
        pr(header.format("Average Costs"))
        pr()
        for account, average_cost in sorted(average_costs.items()):
            for position in average_cost:
                pr(position_line.format('', account, str(position)))
        pr()
        pr()

    # Print the entry itself.
    dcontext = options_map['dcontext']
    pr(header.format("Unbooked Transaction"))
    pr()
    if parsed_entry:
        printer.print_entry(parsed_entry, dcontext, render_weights=True, file=oss)
    pr()

    pr(header.format("Transaction"))
    pr()
    printer.print_entry(entry, dcontext, render_weights=True, file=oss)
    pr()

    if isinstance(entry, data.Transaction):
        pr(header.format("Residual and Tolerances"))
        pr()

        # Print residuals.
        residual = interpolate.compute_residual(entry.postings)
        if not residual.is_empty():
            # Note: We render the residual at maximum precision, for debugging.
            pr('Residual: {}'.format(residual))

        # Dump the tolerances used.
        tolerances = interpolate.infer_tolerances(entry.postings, options_map)
        if tolerances:
            pr('Tolerances: {}'.format(
                ', '.join('{}={}'.format(key, value)
                          for key, value in sorted(tolerances.items()))))

        # Compute the total cost basis.
        cost_basis = inventory.Inventory(
            pos for pos in entry.postings if pos.cost is not None
        ).reduce(convert.get_cost)
        if not cost_basis.is_empty():
            pr('Basis: {}'.format(cost_basis))
        pr()
        pr()

    # Print the context after.
    pr(header.format("Balances after transaction"))
    pr()
    for account in accounts:
        positions = balance_after[account].get_positions()
        for position in positions:
            changed = (account, hash(position)) not in before_hashes
            print(position_line.format('*' if changed else '', account, str(position)),
                  file=oss)
        if not positions:
            pr(position_line.format('', account, ''))
        pr()

    return oss.getvalue()

def render_file_context(entries, options_map, filename, lineno):
    """Render the context before and after a particular transaction is applied.

    Args:
      entries: A list of directives.
      options_map: A dict of options, as produced by the parser.
      filename: A string, the name of the file from which the transaction was parsed.
      lineno: An integer, the line number in the file the transaction was parsed from.
    Returns:
      A multiline string of text, which consists of the context before the
      transaction is applied, the transaction itself, and the context after it
      is applied. You can just print that, it is in form that is intended to be
      consumed by the user.
    """
    # Find the closest entry.
    closest_entry = data.find_closest(entries, filename, lineno)
    if closest_entry is None:
        raise SystemExit("No entry could be found before {}:{}".format(filename, lineno))

    # Run just the parser stage (no booking nor interpolation, which would
    # remove the postings) on the input file to produced the corresponding
    # unbooked transaction, so that we can get the list of accounts.
    if path.exists(filename):
        parsed_entries, _, __= parser.parse_file(filename)

        # Note: We cannot bisect as we cannot rely on sorting behavior from the parser.
        lineno = closest_entry.meta['lineno']
        closest_parsed_entries = [parsed_entry
                                  for parsed_entry in parsed_entries
                                  if parsed_entry.meta['lineno'] == lineno]
        if len(closest_parsed_entries) != 1:
            # This is an internal error, this should never occur.
            raise RuntimeError(
                "Parsed entry corresponding to real entry not found in original filename.")
        closest_parsed_entry = next(iter(closest_parsed_entries))
    else:
        closest_parsed_entry = None

    return render_entry_context(entries, options_map, closest_entry, closest_parsed_entry)

def valid_account_regexp(options):
    """Build a regexp to validate account names from the options.

    Args:
      options: A dict of options, as per beancount.parser.options.
    Returns:
      A string, a regular expression that will match all account names.
    """
    names = map(options.__getitem__, ('name_assets',
                                      'name_liabilities',
                                      'name_equity',
                                      'name_income',
                                      'name_expenses'))

    # Replace the first term of the account regular expression with the specific
    # names allowed under the options configuration. This code is kept in sync
    # with {5672c7270e1e}.
    return regex.compile("(?:{})(?:{}{})+".format('|'.join(names),
                                                  account.sep,
                                                  account.ACC_COMP_NAME_RE))

def check_parser_source_files(parser_module: types.ModuleType):
    """Check the extension module's source hash and issue a warning if the
    current source differs from that of the module.

    If the source files aren't located in the Python source directory, ignore
    the warning, we're probably running this from an installed based, in which
    case we don't need to check anything (this check is useful only for people
    running directly from source).
    """
    parser_source_hash = hash_parser_source_files()
    if parser_source_hash is None:
        return
    if parser_module.SOURCE_HASH and parser_module.SOURCE_HASH != parser_source_hash:
        warnings.warn(
            ("The Beancount parser C extension module is out-of-date ('{}' != '{}'). "
             "You need to rebuild.").format(parser_module.SOURCE_HASH, parser_source_hash))

def gen_include():
    """Generate an include file for the parser source hash."""
    return textwrap.dedent("""\
      #ifndef __BEANCOUNT_PARSER_PARSE_SOURCE_HASH_H__
      #define __BEANCOUNT_PARSER_PARSE_SOURCE_HASH_H__

      #define PARSER_SOURCE_HASH {source_hash}

      #endif // __BEANCOUNT_PARSER_PARSE_SOURCE_HASH_H__
    """.format(source_hash=hash_parser_source_files()))

def hash_parser_source_files():
    """Compute a unique hash of the parser's Python code in order to bake that into
    the extension module. This is used at load-time to verify that the extension
    module and the corresponding Python codes match each other. If not, it
    issues a warning that you should rebuild your extension module.

    Returns:
      A string, the hexadecimal unique hash of relevant source code that should
      trigger a recompilation.
    """
    md5 = hashlib.md5()
    for filename in PARSER_SOURCE_FILES:
        fullname = path.join(path.dirname(__file__), filename)
        if not path.exists(fullname):
            return None
        with open(fullname, 'rb') as file:
            md5.update(file.read())
    # Note: Prepend a character in front of the hash because under Windows MSDEV
    # removes escapes, and if the hash starts with a number it fails to
    # recognize this is a string. A small compromise for portability.
    return md5.hexdigest()

def lex_iter(file, builder=None):
    """An iterator that yields all the tokens in the given file.

    Args:
      file: A string, the filename to run the lexer on, or a file object.
      builder: A builder of your choice. If not specified, a LexBuilder is
        used and discarded (along with its errors).
    Yields:
      All the tokens in the input file as ``(token, lineno, text,
      value)`` tuples where ``token`` is a string representing the
      token kind, ``lineno`` is the line number in the input file
      where the token was matched, ``mathed`` is a bytes object
      containing the exact text matched, and ``value`` is the semantic
      value of the token or None.
    """
    with contextlib.ExitStack() as ctx:
        # It would be more appropriate here to check for io.RawIOBase but
        # that does not work for io.BytesIO despite it implementing the
        # readinto() method.
        if not isinstance(file, io.IOBase):
            file = ctx.enter_context(open(file, 'rb'))
        if builder is None:
            builder = LexBuilder()
        parser = _parser.Parser(builder)
        yield from parser.lex(file)

def lex_iter_string(string, builder=None, **kwargs):
    """An iterator that yields all the tokens in the given string.

    Args:
      string: a str or bytes, the contents of the ledger to be parsed.
    Returns:
      An iterator, see ``lex_iter()`` for details.
    """
    if not isinstance(string, bytes):
        string = string.encode('utf8')
    file = io.BytesIO(string)
    yield from lex_iter(file, builder=builder, **kwargs)

def Opt(name, default_value,
        example_value=UNSET,
        converter=None,
        deprecated=False,
        alias=None):
    """Alternative constructor for OptDesc, with default values.

    Args:
      name: See OptDesc.
      default_value: See OptDesc.
      example_value: See OptDesc.
      converter: See OptDesc.
      deprecated: See OptDesc.
      alias: See OptDesc.
    Returns:
      An instance of OptDesc.
    """
    if example_value is UNSET:
        example_value = default_value
    return OptDesc(name, default_value, example_value, converter, deprecated, alias)

def get_account_types(options):
    """Extract the account type names from the parser's options.

    Args:
      options: a dict of ledger options.
    Returns:
      An instance of AccountTypes, that contains all the prefixes.
    """
    return account_types.AccountTypes(
        *[options[key]
          for key in ("name_assets",
                      "name_liabilities",
                      "name_equity",
                      "name_income",
                      "name_expenses")])

def get_current_accounts(options):
    """Return account names for the current earnings and conversion accounts.

    Args:
      options: a dict of ledger options.
    Returns:
      A tuple of 2 account objects, one for booking current earnings, and one
      for current conversions.
    """
    equity = options['name_equity']
    account_current_earnings = account.join(equity,
                                            options['account_current_earnings'])
    account_current_conversions = account.join(equity,
                                               options['account_current_conversions'])
    return (account_current_earnings,
            account_current_conversions)

def get_previous_accounts(options):
    """Return account names for the previous earnings, balances and conversion accounts.

    Args:
      options: a dict of ledger options.
    Returns:
      A tuple of 3 account objects, for booking previous earnings,
      previous balances, and previous conversions.
    """
    equity = options['name_equity']
    account_previous_earnings = account.join(equity,
                                             options['account_previous_earnings'])
    account_previous_balances = account.join(equity,
                                             options['account_previous_balances'])
    account_previous_conversions = account.join(equity,
                                                options['account_previous_conversions'])
    return (account_previous_earnings,
            account_previous_balances,
            account_previous_conversions)

def get_unrealized_account(options):
    """Return the full account name for the unrealized account.

    Args:
      options: a dict of ledger options.
    Returns:
      A tuple of 2 account objects, one for booking current earnings, and one
      for current conversions.
    """
    income = options['name_income']
    return  account.join(income, options['account_unrealized_gains'])

def list_options():
    """Produce a formatted text of the available options and their description.

    Returns:
      A string, formatted nicely to be printed in 80 columns.
    """
    oss = io.StringIO()
    for group in PUBLIC_OPTION_GROUPS:
        for desc in group.options:
            oss.write('option "{}" "{}"\n'.format(desc.name, desc.example_value))
            if desc.deprecated:
                oss.write(textwrap.fill(
                    "THIS OPTION IS DEPRECATED: {}".format(desc.deprecated),
                    initial_indent="  ",
                    subsequent_indent="  "))
                oss.write('\n\n')
        description = ' '.join(line.strip()
                               for line in group.description.strip().splitlines())
        oss.write(textwrap.fill(description,
                                initial_indent='  ',
                                subsequent_indent='  '))
        oss.write('\n')

        if isinstance(desc.default_value, (list, dict, set)):
            oss.write('\n')
            oss.write('  (This option may be supplied multiple times.)\n')

        oss.write('\n\n')

    return oss.getvalue()

def options_validate_booking_method(value):
    """Validate a booking method name.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    try:
        return data.Booking[value]
    except KeyError as exc:
        raise ValueError(str(exc)) from exc

def options_validate_boolean(value):
    """Validate a boolean option.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    return value.lower() in ('1', 'true', 'yes')

def options_validate_plugin(value):
    """Validate the plugin option.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    # Process the 'plugin' option specially: accept an optional
    # argument from it. NOTE: We will eventually phase this out and
    # replace it by a dedicated 'plugin' directive.
    match = re.match('(.*):(.*)', value)
    if match:
        plugin_name, plugin_config = match.groups()
    else:
        plugin_name, plugin_config = value, None
    return (plugin_name, plugin_config)

def options_validate_processing_mode(value):
    """Validate the options processing mode.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    if value not in ('raw', 'default'):
        raise ValueError("Invalid value '{}'".format(value))
    return value

def options_validate_tolerance(value):
    """Validate the tolerance option.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    return D(value)

def options_validate_tolerance_map(value):
    """Validate an option with a map of currency/tolerance pairs in a string.

    Args:
      value: A string, the value provided as option.
    Returns:
      The new value, converted, if the conversion is successful.
    Raises:
      ValueError: If the value is invalid.
    """
    # Process the setting of a key-value, whereby the value is a Decimal
    # representation.
    match = re.match('(.*):(.*)', value)
    if not match:
        raise ValueError("Invalid value '{}'".format(value))
    currency, tolerance_str = match.groups()
    return (currency, D(tolerance_str))

def is_entry_incomplete(entry):
    """Detect the presence of elided amounts in Transactions.

    Args:
      entries: A directive.
    Returns:
      A boolean, true if there are some missing portions of any postings found.
    """
    if isinstance(entry, data.Transaction):
        if any(is_posting_incomplete(posting) for posting in entry.postings):
            return True
    return False

def is_posting_incomplete(posting):
    """Detect the presence of any elided amounts in a Posting.

    If any of the possible amounts are missing, this returns True.

    Args:
      entries: A directive.
    Returns:
      A boolean, true if there are some missing portions of any postings found.
    """
    units = posting.units
    if (units is MISSING or
        units.number is MISSING or
        units.currency is MISSING):
        return True
    price = posting.price
    if (price is MISSING or
        price is not None and (price.number is MISSING or
                               price.currency is MISSING)):
        return True
    cost = posting.cost
    if cost is not None and (cost.number_per is MISSING or
                             cost.number_total is MISSING or
                             cost.currency is MISSING):
        return True
    return False

def parse_doc(expect_errors=False, allow_incomplete=False):
    """Factory of decorators that parse the function's docstring as an argument.

    Note that the decorators thus generated only run the parser on the tests,
    not the loader, so is no validation, balance checks, nor plugins applied to
    the parsed text.

    Args:
      expect_errors: A boolean or None, with the following semantics,
        True: Expect errors and fail if there are none.
        False: Expect no errors and fail if there are some.
        None: Do nothing, no check.
      allow_incomplete: A boolean, if true, allow incomplete input. Otherwise
        barf if the input would require interpolation. The default value is set
        not to allow it because we want to minimize the features tests depend on.
    Returns:
      A decorator for test functions.
    """
    def decorator(fun):
        """A decorator that parses the function's docstring as an argument.

        Args:
          fun: the function object to be decorated.
        Returns:
          A decorated test function.
        """
        filename = inspect.getfile(fun)
        lines, lineno = inspect.getsourcelines(fun)

        # Skip over decorator invocation and function definition. This
        # is imperfect as it assumes that each consumes exactly one
        # line, but this is by far the most common case, and this is
        # mainly used in test, thus it is good enough.
        lineno += 2

        @functools.wraps(fun)
        def wrapper(self):
            assert fun.__doc__ is not None, (
                "You need to insert a docstring on {}".format(fun.__name__))
            entries, errors, options_map = parse_string(fun.__doc__,
                                                        report_filename=filename,
                                                        report_firstline=lineno,
                                                        dedent=True)

            if not allow_incomplete and any(is_entry_incomplete(entry)
                                            for entry in entries):
                self.fail("parse_doc() may not use interpolation.")

            if expect_errors is not None:
                if expect_errors is False and errors:
                    oss = io.StringIO()
                    printer.print_errors(errors, file=oss)
                    self.fail("Unexpected errors found:\n{}".format(oss.getvalue()))
                elif expect_errors is True and not errors:
                    self.fail("Expected errors, none found:")

            return fun(self, entries, errors, options_map)

        return wrapper

    return decorator

def parse_file(file, report_filename=None, report_firstline=1,
               encoding=None, debug=False, **kw):
    """Parse a beancount input file and return Ledger with the list of
    transactions and tree of accounts.

    Args:
      file: file object or path to the file to be parsed.
      kw: a dict of keywords to be applied to the C parser.
    Returns:
      A tuple of (
        list of entries parsed in the file,
        list of errors that were encountered during parsing, and
        a dict of the option values that were parsed from the file.)
    """
    if encoding is not None and codecs.lookup(encoding).name != 'utf-8':
        raise ValueError('Only UTF-8 encoded files are supported.')
    with contextlib.ExitStack() as ctx:
        if file == '-':
            file = sys.stdin.buffer
        # It would be more appropriate here to check for io.RawIOBase but
        # that does not work for io.BytesIO despite it implementing the
        # readinto() method.
        elif not isinstance(file, io.IOBase):
            file = ctx.enter_context(open(file, 'rb'))
        builder = grammar.Builder()
        parser = _parser.Parser(builder, debug=debug)
        parser.parse(file, filename=report_filename, lineno=report_firstline, **kw)
    return builder.finalize()

def parse_many(string, level=0):
    """Parse a string with a snippet of Beancount input and replace vars from caller.

    Args:
      string: A string with some Beancount input.
      level: The number of extra stacks to ignore.
    Returns:
      A list of entries.
    Raises:
      AssertionError: If there are any errors.
    """
    # Get the locals in the stack for the callers and produce the final text.
    frame = inspect.stack()[level+1]
    varkwds = frame[0].f_locals
    input_string = textwrap.dedent(string.format(**varkwds))

    # Parse entries and check there are no errors.
    entries, errors, __ = parse_string(input_string)
    assert not errors

    return entries

def parse_one(string):
    """Parse a string with single Beancount directive and replace vars from caller.

    Args:
      string: A string with some Beancount input.
      level: The number of extra stacks to ignore.
    Returns:
      A list of entries.
    Raises:
      AssertionError: If there are any errors.
    """
    entries = parse_many(string, level=1)
    assert len(entries) == 1
    return entries[0]

def parse_string(string, report_filename=None, dedent=False, **kw):
    """Parse a beancount input file and return Ledger with the list of
    transactions and tree of accounts.

    Args:
      string: A string, the contents to be parsed instead of a file's.
      report_filename: A string, the source filename from which this string
        has been extracted, if any. This is stored in the metadata of the
        parsed entries.
      dedent: Whether to run textwrap.dedent() on the string before parsing.
      **kw: See parse.c.
    Return:
      Same as the output of parse_file().
    """
    if dedent:
        string = textwrap.dedent(string)
    if isinstance(string, str):
        string = string.encode('utf8')
    if report_filename is None:
        report_filename = '<string>'
    file = io.BytesIO(string)
    return parse_file(file, report_filename=report_filename, **kw)

def align_position_strings(strings):
    """A helper used to align rendered amounts positions to their first currency
    character (an uppercase letter). This class accepts a list of rendered
    positions and calculates the necessary width to render them stacked in a
    column so that the first currency word aligns. It does not go beyond that
    (further currencies, e.g. for the price or cost, are not aligned).

    This is perhaps best explained with an example. The following positions will
    be aligned around the column marked with '^':

              45 HOOL {504.30 USD}
               4 HOOL {504.30 USD, 2014-11-11}
            9.95 USD
       -22473.32 CAD @ 1.10 USD
                 ^

    Strings without a currency character will be rendered flush left.

    Args:
      strings: A list of rendered position or amount strings.
    Returns:
      A pair of a list of aligned strings and the width of the aligned strings.
    """
    # Maximum length before the alignment character.
    max_before = 0
    # Maximum length after the alignment character.
    max_after = 0
    # Maximum length of unknown strings.
    max_unknown = 0

    string_items = []
    search = re.compile('[A-Z]').search
    for string in strings:
        match = search(string)
        if match:
            index = match.start()
            if index != 0:
                max_before = max(index, max_before)
                max_after = max(len(string) - index, max_after)
                string_items.append((index, string))
                continue
        # else
        max_unknown = max(len(string), max_unknown)
        string_items.append((None, string))

    # Compute formatting string.
    max_total = max(max_before + max_after, max_unknown)
    max_after_prime = max_total - max_before
    fmt = "{{:>{0}}}{{:{1}}}".format(max_before, max_after_prime).format
    fmt_unknown = "{{:<{0}}}".format(max_total).format

    # Align the strings and return them.
    # pylint: disable=format-string-without-interpolation
    aligned_strings = []
    for index, string in string_items:
        # pylint: disable=format-string-without-interpolation
        if index is not None:
            string = fmt(string[:index], string[index:])
        else:
            string = fmt_unknown(string)
        aligned_strings.append(string)

    return aligned_strings, max_total

def format_entry(entry, dcontext=None, render_weights=False, prefix=None,
                 write_source=False):
    """Format an entry into a string in the same input syntax the parser accepts.

    Args:
      entry: An entry instance.
      dcontext: An instance of DisplayContext used to format the numbers.
      render_weights: A boolean, true to render the weights for debugging.
      write_source: If true a source file and line number will be written for
        each entry in a format interpretable as a message location for Emacs,
        VSCode or other editors. As this is for
        "debugging" purposes, this information will be commented out by a
        semicolon.
    Returns:
      A string, the formatted entry.
    """
    return EntryPrinter(dcontext, render_weights, prefix=prefix,
                        write_source=write_source)(entry)

def format_error(error):
    """Given an error objects, return a formatted string for it.

    Args:
      error: a namedtuple objects representing an error. It has to have an
        'entry' attribute that may be either a single directive object or a
        list of directive objects.
    Returns:
      A string, the errors rendered.
    """
    oss = io.StringIO()
    oss.write('{} {}\n'.format(render_source(error.source), error.message))
    if error.entry is not None:
        entries = error.entry if isinstance(error.entry, list) else [error.entry]
        error_string = '\n'.join(format_entry(entry) for entry in entries)
        oss.write('\n')
        oss.write(textwrap.indent(error_string, '   '))
        oss.write('\n')
    return oss.getvalue()

def print_entries(entries, dcontext=None, render_weights=False, file=None, prefix=None,
                  write_source=False):
    """A convenience function that prints a list of entries to a file.

    Args:
      entries: A list of directives.
      dcontext: An instance of DisplayContext used to format the numbers.
      render_weights: A boolean, true to render the weights for debugging.
      file: An optional file object to write the entries to.
      prefix: User-specific prefix for custom indentation (for Fava).
      write_source: If true a source file and line number will be written for
        each entry in a format interpretable as a message location for Emacs,
        VSCode or other editors. This is usefull for "debugging" peurposes,
        especially in a multi-file setup
    """
    assert isinstance(entries, list), "Entries is not a list: {}".format(entries)
    output = file or (codecs.getwriter("utf-8")(sys.stdout.buffer)
                      if hasattr(sys.stdout, 'buffer') else
                      sys.stdout)

    if prefix:
        output.write(prefix)
    previous_type = type(entries[0]) if entries else None
    eprinter = EntryPrinter(dcontext, render_weights, write_source=write_source)
    for entry in entries:
        # Insert a newline between transactions and between blocks of directives
        # of the same type.
        entry_type = type(entry)
        if (entry_type in (data.Transaction, data.Commodity) or
            entry_type is not previous_type or write_source):
            output.write('\n')
            previous_type = entry_type

        string = eprinter(entry)
        output.write(string)

def print_entry(entry, dcontext=None, render_weights=False, file=None,
                write_source=False):
    """A convenience function that prints a single entry to a file.

    Args:
      entry: A directive entry.
      dcontext: An instance of DisplayContext used to format the numbers.
      render_weights: A boolean, true to render the weights for debugging.
      file: An optional file object to write the entries to.
      write_source: If true a source file and line number will be written for
        each entry in a format interpretable as a message location for Emacs,
        VSCode or other editors. This is usefull for "debugging" purposes,
        especially in a multi-file setup
    """
    # TODO(blais): DO remove this now, it's a huge annoyance not to be able to
    # print in-between other statements.
    output = file or (codecs.getwriter("utf-8")(sys.stdout.buffer)
                      if hasattr(sys.stdout, 'buffer') else
                      sys.stdout)
    output.write(format_entry(entry, dcontext, render_weights,
                              write_source=write_source))
    output.write('\n')

def print_error(error, file=None):
    """A convenience function that prints a single error to a file.

    Args:
      error: An error object.
      file: An optional file object to write the errors to.
    """
    output = file or sys.stdout
    output.write(format_error(error))
    output.write('\n')

def print_errors(errors, file=None, prefix=None):
    """A convenience function that prints a list of errors to a file.

    Args:
      errors: A list of errors.
      file: An optional file object to write the errors to.
    """
    output = file or sys.stdout
    if prefix:
        output.write(prefix)
    for error in errors:
        output.write(format_error(error))
        output.write('\n')

def render_flag(inflag: Optional[str]) -> str:
    """Render a flag, which can be None, a symbol of a character to a string."""
    if not inflag:
        return ''
    return inflag

def render_source(meta):
    """Render the source for errors in a way that it will be both detected by
    Emacs and align and rendered nicely.

    Args:
      meta: A dict with the metadata.
    Returns:
      A string, rendered to be interpretable as a message location for Emacs or
      other editors.
    """
    return '{}:{:8}'.format(meta['filename'], '{}:'.format(meta['lineno']))

def compute_version_string(version, changeset, timestamp):
    """Compute a version string from the changeset and timestamp baked in the parser.

    Args:
      version: A string, the version number.
      changeset: A string, a version control string identifying the commit of the version.
      timestamp: An integer, the UNIX epoch timestamp of the changeset.
    Returns:
      A human-readable string for the version.
    """
    # Shorten changeset.
    if changeset:
        if re.match('hg:', changeset):
            changeset = changeset[:15]
        elif re.match('git:', changeset):
            changeset = changeset[:12]

    # Convert timestamp to a date.
    date = None
    if timestamp > 0:
        date = datetime.datetime.fromtimestamp(timestamp, datetime.UTC).date()

    version = 'Beancount {}'.format(version)
    if changeset or date:
        version = '{} ({})'.format(
            version, '; '.join(map(str, filter(None, [changeset, date]))))

    return version