src/budget_sync/main.py

   1 import os
   2 import re
   3 import sys
   4 import json
   5 from typing import Optional
   6 from budget_sync.ordered_set import OrderedSet
   7 from budget_sync.write_budget_csv import write_budget_csv
   8 from bugzilla import Bugzilla
   9 import logging
  10 import argparse
  11 from pathlib import Path
  12 from budget_sync.util import all_bugs, tty_out
  13 from budget_sync.config import Config, ConfigParseError
  14 from budget_sync.budget_graph import BudgetGraph, PaymentSummary
  15 from budget_sync.write_budget_markdown import (write_budget_markdown,
  16                                                markdown_for_person)
  17
  18
  19 def main():
  20     parser = argparse.ArgumentParser(
  21         description="Check for errors in "
  22         "Libre-SOC's style of budget tracking in Bugzilla.")
  23     parser.add_argument(
  24         "-c", "--config", type=argparse.FileType('r'),
  25         required=True, help="The path to the configuration TOML file",
  26         dest="config", metavar="<path/to/budget-sync-config.toml>")
  27     parser.add_argument(
  28         "-o", "--output-dir", type=Path, default=None,
  29         help="The path to the output directory, will be created if it "
  30         "doesn't exist",
  31         dest="output_dir", metavar="<path/to/output/dir>")
  32     parser.add_argument('--subset',
  33                         help="write the output for this subset of bugs",
  34                         metavar="<bug-id>,<bug-id>,...")
  35     parser.add_argument('--subset-person',
  36                         help="write the output for this person",
  37                         dest="person")
  38     parser.add_argument('--username',
  39                         help="Log in with this Bugzilla username")
  40     parser.add_argument('--password',
  41                         help="Log in with this Bugzilla password")
  42     parser.add_argument('--comments', action='store_true',
  43                         help="Put JSON into comments")
  44     args = parser.parse_args()
  45     try:
  46         with args.config as config_file:
  47             config = Config.from_file(config_file)
  48     except (IOError, ConfigParseError) as e:
  49         logging.error("Failed to parse config file: %s", e)
  50         return
  51     logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
  52     print("```")  # for using the output as markdown
  53     bz = Bugzilla(config.bugzilla_url)
  54     if args.username:
  55         logging.debug("logging in...")
  56         bz.interactive_login(args.username, args.password)
  57     logging.debug("Connected to Bugzilla")
  58     budget_graph = BudgetGraph(all_bugs(bz), config)
  59     for error in budget_graph.get_errors():
  60         logging.error("%s", error)
  61     if args.person or args.subset:
  62         if not args.person:
  63             logging.fatal("must use --subset-person with --subset option")
  64             sys.exit(1)
  65         print_markdown_for_person(budget_graph, config,
  66                                   args.person, args.subset)
  67         return
  68     if args.output_dir is not None:
  69         write_budget_markdown(budget_graph, args.output_dir)
  70         write_budget_csv(budget_graph, args.output_dir)
  71     summarize_milestones(budget_graph)
  72     print("```")  # for using the output as markdown
  73     json_milestones(budget_graph, args.comments, args.output_dir)
  74
  75
  76 def print_markdown_for_person(budget_graph: BudgetGraph, config: Config,
  77                               person_str: str, subset_str: Optional[str]):
  78     person = config.all_names.get(person_str)
  79     if person is None:
  80         logging.fatal("--subset-person: unknown person: %s", person_str)
  81         sys.exit(1)
  82     nodes_subset = None
  83     if subset_str:
  84         nodes_subset = OrderedSet()
  85         for bug_id in re.split(r"[\s,]+", subset_str):
  86             try:
  87                 node = budget_graph.nodes[int(bug_id)]
  88             except (ValueError, KeyError):
  89                 logging.fatal("--subset: unknown bug: %s", bug_id)
  90                 sys.exit(1)
  91             nodes_subset.add(node)
  92     print(markdown_for_person(budget_graph, person, nodes_subset))
  93
  94
  95 def print_budget_then_children(indent, nodes, bug_id):
  96     """recursive indented printout of budgets
  97     """
  98
  99     bug = nodes[bug_id]
 100     b_incl = str(bug.fixed_budget_including_subtasks)
 101     b_excl = str(bug.fixed_budget_excluding_subtasks)
 102     s_incl = str(bug.submitted_including_subtasks)
 103     p_incl = str(bug.paid_including_subtasks)
 104     if b_incl == s_incl and b_incl == p_incl:
 105         descr = "(s+p)"
 106     elif b_incl == s_incl:
 107         descr = "(s) p %s" % p_incl
 108     elif b_incl == p_incl:
 109         descr = "(p) s %s" % s_incl
 110     elif s_incl == p_incl:
 111         descr = " s,p  %s" % (p_incl)
 112     else:
 113         descr = "s %s p %s" % (s_incl, p_incl)
 114     excl_desc = "               "
 115     if b_incl != b_excl:
 116         excl_desc = "excltasks %6s" % b_excl
 117     print("bug #%5d %s budget %6s %s %s" %
 118           (bug.bug.id, ' |   ' * indent,
 119            b_incl,
 120            excl_desc,
 121            descr
 122            ))
 123     # print(repr(bug))
 124
 125     for child in bug.immediate_children:
 126         if (str(child.budget_including_subtasks) == "0" and
 127                 str(child.budget_excluding_subtasks) == "0"):
 128             continue
 129         print_budget_then_children(indent+1, nodes, child.bug.id)
 130
 131
 132 def summarize_milestones(budget_graph: BudgetGraph):
 133     for milestone, payments in budget_graph.milestone_payments.items():
 134         summary = PaymentSummary(payments)
 135         print(f"{milestone.identifier}")
 136         print(f"\t{summary.total} submitted: "
 137               f"{summary.total_submitted} paid: {summary.total_paid}")
 138         not_submitted = summary.get_not_submitted()
 139         if not_submitted:
 140             print("not submitted", not_submitted)
 141
 142         # and one to display people
 143         for person in budget_graph.milestone_people[milestone]:
 144             print(f"\t%-30s - %s" % (person.identifier, person.full_name))
 145         print()
 146
 147     # now do trees
 148     for milestone, payments in budget_graph.milestone_payments.items():
 149         print("%s %d" % (milestone.identifier, milestone.canonical_bug_id))
 150         print_budget_then_children(0, budget_graph.nodes,
 151                                    milestone.canonical_bug_id)
 152         print()
 153
 154
 155 def json_milestones(budget_graph: BudgetGraph, add_comments: bool,
 156                     output_dir: Path):
 157     """reports milestones as json format
 158     """
 159     bug_comments_map = {}
 160     if add_comments:
 161         need_set = set()
 162         bugzilla = None
 163         for nodes in budget_graph.assigned_nodes_for_milestones.values():
 164             for node in nodes:
 165                 need_set.add(node.bug.id)
 166                 bugzilla = node.bug.bugzilla
 167         need_list = sorted(need_set)
 168         total = len(need_list)
 169         with tty_out() as term:
 170             step = 100
 171             i = 0
 172             while i < total:
 173                 cur_need = need_list[i:i + step]
 174                 stop = i + len(cur_need)
 175                 print(f"loading comments {i}:{stop} of {total}",
 176                       flush=True, file=term)
 177                 comments = bugzilla.get_comments(cur_need)['bugs']
 178                 if len(comments) < len(cur_need) and len(cur_need) > 1:
 179                     step = max(1, step // 2)
 180                     print(f"failed, trying smaller step of {step}",
 181                           flush=True, file=term)
 182                     continue
 183                 bug_comments_map.update(comments)
 184                 i += len(cur_need)
 185     for milestone, payments in budget_graph.milestone_payments.items():
 186         summary = PaymentSummary(payments)
 187         # and one to display people
 188         ppl = []
 189         for person in budget_graph.milestone_people[milestone]:
 190             p = {'name': person.full_name, 'email': person.email}
 191             ppl.append(p)
 192
 193         tasks = []
 194         canonical = budget_graph.nodes[milestone.canonical_bug_id]
 195         for child in canonical.immediate_children:
 196             milestones = []
 197             # include the task itself as a milestone
 198             for st in list(child.children()) + [child]:
 199                 amount = st.fixed_budget_excluding_subtasks.int()
 200                 if amount == 0:  # skip anything at zero
 201                     continue
 202                 # if "task itself" then put the milestone as "wrapup"
 203                 if st.bug == child.bug:
 204                     description = 'wrapup'
 205                     intro = []
 206                 else:
 207                     # otherwise create a description and get comment #0
 208                     description = "%d %s" % (st.bug.id, st.bug.summary)
 209                     # add parent and MoU top-level
 210                     parent_id = st.parent.bug.id
 211                     if parent_id != child.bug.id:
 212                         description += "\n(Sub-sub-task of %d)" % parent_id
 213                 task = {'description': description,
 214                         'amount': amount,
 215                         }
 216                 #mou_bug = st.closest_bug_in_mou
 217                 # if mou_bug is not None:
 218                 #    task['mou_task'] = mou_bug.bug.id
 219                 milestones.append(task)
 220             # create MoU task: get comment #0
 221             intro = []
 222             comment = "%s\n " % child.bug_url
 223             if add_comments:
 224                 comment += "\n"
 225                 comments = bug_comments_map[str(child.bug.id)]['comments']
 226                 lines = comments[0]['text'].splitlines()
 227                 for i, line in enumerate(lines):
 228                     # look for a line with only 4 or more `-` and blank lines
 229                     # before and after, like so:
 230                     # ```
 231                     #
 232                     # -----
 233                     #
 234                     # ```
 235                     if len(line) >= 4 and line == "-" * len(line) \
 236                         and i >= 1 and lines[i - 1] == "" and (
 237                             i > len(lines) or lines[i + 1] == ""):
 238                         lines[i:] = []
 239                         break
 240                     if line == "<From here down doesn't go in MOU>":
 241                         lines[i:] = []
 242                         break
 243                 comment += "\n".join(lines)
 244             intro.append(comment)
 245             # print (description, intro)
 246             # sys.stdout.flush()
 247             task = {'title': "%d %s" % (child.bug.id, child.bug.summary),
 248                     'intro': intro,
 249                     'amount': child.fixed_budget_including_subtasks.int(),
 250                     'url': "{{ %s }} " % child.bug_url,
 251                     'milestones': milestones
 252                     }
 253             tasks.append(task)
 254
 255         d = {'participants': ppl,
 256              'preamble': '',
 257              'type': 'Group',
 258              'url': canonical.bug_url,
 259              'plan': {'intro': [''],
 260                       'tasks': tasks,
 261                       'rfp_secret': '',
 262                       }
 263              }
 264
 265         output_file = output_dir / f"report.{milestone.identifier}.json"
 266         output_file.write_text(json.dumps(d, indent=2), encoding="utf-8")
 267
 268
 269 if __name__ == "__main__":
 270     main()