src/budget_sync/main.py

   1 import os
   2 import re
   3 import sys
   4 import json
   5 from typing import Optional
   6 from budget_sync.ordered_set import OrderedSet
   7 from budget_sync.write_budget_csv import write_budget_csv
   8 from bugzilla import Bugzilla
   9 import logging
  10 import argparse
  11 from pathlib import Path
  12 from budget_sync.util import all_bugs, tty_out
  13 from budget_sync.config import Config, ConfigParseError
  14 from budget_sync.budget_graph import BudgetGraph, PaymentSummary
  15 from budget_sync.write_budget_markdown import (write_budget_markdown,
  16                                                markdown_for_person)
  17
  18 def spc(s):
  19     return s.replace(" ", "&nbsp;")
  20
  21
  22 def main():
  23     parser = argparse.ArgumentParser(
  24         description="Check for errors in "
  25         "Libre-SOC's style of budget tracking in Bugzilla.")
  26     parser.add_argument(
  27         "-c", "--config", type=argparse.FileType('r'),
  28         required=True, help="The path to the configuration TOML file",
  29         dest="config", metavar="<path/to/budget-sync-config.toml>")
  30     parser.add_argument(
  31         "-o", "--output-dir", type=Path, default=None,
  32         help="The path to the output directory, will be created if it "
  33         "doesn't exist",
  34         dest="output_dir", metavar="<path/to/output/dir>")
  35     parser.add_argument('--subset',
  36                         help="write the output for this subset of bugs",
  37                         metavar="<bug-id>,<bug-id>,...")
  38     parser.add_argument('--subset-person',
  39                         help="write the output for this person",
  40                         dest="person")
  41     parser.add_argument('--username',
  42                         help="Log in with this Bugzilla username")
  43     parser.add_argument('--password',
  44                         help="Log in with this Bugzilla password")
  45     parser.add_argument('--comments', action='store_true',
  46                         help="Put JSON into comments")
  47     parser.add_argument('--detail', action='store_true',
  48                         help="Add detail to report (links description people)")
  49     args = parser.parse_args()
  50     try:
  51         with args.config as config_file:
  52             config = Config.from_file(config_file)
  53     except (IOError, ConfigParseError) as e:
  54         logging.error("Failed to parse config file: %s", e)
  55         return
  56     logging.info("Using Bugzilla instance at %s", config.bugzilla_url)
  57     # download all bugs and create a summary report
  58     reportdir = "./" if args.output_dir is None else str(args.output_dir)
  59     reportdir = "/".join(reportdir.split("/")[:-1]) # strip /mdwn
  60     reportname = reportdir + "/report.mdwn"
  61     with open(reportname, "w") as f:
  62         print("<tt>", file=f)  # for using the output as markdown
  63         bz = Bugzilla(config.bugzilla_url)
  64         if args.username:
  65             logging.debug("logging in...")
  66             bz.interactive_login(args.username, args.password)
  67         logging.debug("Connected to Bugzilla")
  68         budget_graph = BudgetGraph(all_bugs(bz), config)
  69         for error in budget_graph.get_errors():
  70             logging.error("%s", error)
  71         if args.person or args.subset:
  72             if not args.person:
  73                 logging.fatal("must use --subset-person with --subset option")
  74                 sys.exit(1)
  75             print_markdown_for_person(f, budget_graph, config,
  76                                       args.person, args.subset)
  77             return
  78         if args.output_dir is not None:
  79             write_budget_markdown(budget_graph, args.output_dir)
  80             write_budget_csv(budget_graph, args.output_dir)
  81         summarize_milestones(f, budget_graph, detail=args.detail)
  82         print("</tt>", file=f)  # for using the output as markdown
  83     # now create the JSON milestone files for putting into NLnet RFP system
  84     json_milestones(budget_graph, args.comments, args.output_dir)
  85
  86
  87 def print_markdown_for_person(f, budget_graph, config, person_str, subset_str):
  88     person = config.all_names.get(person_str)
  89     if person is None:
  90         logging.fatal("--subset-person: unknown person: %s", person_str)
  91         sys.exit(1)
  92     nodes_subset = None
  93     if subset_str:
  94         nodes_subset = OrderedSet()
  95         for bug_id in re.split(r"[\s,]+", subset_str):
  96             try:
  97                 node = budget_graph.nodes[int(bug_id)]
  98             except (ValueError, KeyError):
  99                 logging.fatal("--subset: unknown bug: %s", bug_id)
 100                 sys.exit(1)
 101             nodes_subset.add(node)
 102     print(markdown_for_person(budget_graph, person, nodes_subset), file=f)
 103
 104
 105 def print_budget_then_children(f, indent, nodes, bug_id, detail=False):
 106     """recursive indented printout of budgets
 107     """
 108
 109     bug = nodes[bug_id]
 110     b_incl = str(bug.fixed_budget_including_subtasks)
 111     b_excl = str(bug.fixed_budget_excluding_subtasks)
 112     s_incl = str(bug.submitted_including_subtasks)
 113     p_incl = str(bug.paid_including_subtasks)
 114     if b_incl == s_incl and b_incl == p_incl:
 115         descr = "(s+p)"
 116     elif b_incl == s_incl:
 117         descr = "(s) p %s" % p_incl
 118     elif b_incl == p_incl:
 119         descr = "(p) s %s" % s_incl
 120     elif s_incl == p_incl:
 121         descr = " s,p  %s" % (p_incl)
 122     else:
 123         descr = "s %s p %s" % (s_incl, p_incl)
 124     excl_desc = "               "
 125     if b_incl != b_excl:
 126         excl_desc = "excltasks %6s" % b_excl
 127     print(spc("bug #%5d %s budget %6s %s %s<br>" %
 128               (bug.bug.id, ' |   ' * indent,
 129                b_incl,
 130                excl_desc,
 131                descr
 132                )), file=f)
 133     if detail:
 134         print(spc("           %s |      (%s)<br>" %
 135                   (' |   ' * indent,
 136                    bug.bug.summary[:40]
 137                   )), file=f)
 138     # print(repr(bug))
 139
 140     for child in bug.immediate_children:
 141         if (str(child.budget_including_subtasks) == "0" and
 142                 str(child.budget_excluding_subtasks) == "0"):
 143             continue
 144         print_budget_then_children(f, indent+1, nodes, child.bug.id, detail)
 145
 146
 147 def summarize_milestones(f, budget_graph, detail=False):
 148     for milestone, payments in budget_graph.milestone_payments.items():
 149         summary = PaymentSummary(payments)
 150         print(f"{milestone.identifier}<br>", file=f)
 151         print(f"    {summary.total} submitted: "
 152               f"{summary.total_submitted} paid: {summary.total_paid}<br>",
 153               file=f)
 154         not_submitted = summary.get_not_submitted()
 155         if not_submitted:
 156             print("not submitted %s<br>" % not_submitted, file=f)
 157
 158         # and one to display people
 159         for person in budget_graph.milestone_people[milestone]:
 160             print(spc("    %-30s - %s<br>" % (person.identifier,
 161                                             person.full_name)),
 162                   file=f)
 163         print("<br>", file=f)
 164
 165     # now do trees
 166     for milestone, payments in budget_graph.milestone_payments.items():
 167         print("%s %d<br>" % (milestone.identifier, milestone.canonical_bug_id),
 168               file=f)
 169         print_budget_then_children(f, 0, budget_graph.nodes,
 170                                    milestone.canonical_bug_id, detail)
 171         print("<br>", file=f)
 172
 173
 174 def json_milestones(budget_graph: BudgetGraph, add_comments: bool,
 175                     output_dir: Path):
 176     """reports milestones as json format
 177     """
 178     bug_comments_map = {}
 179     if add_comments:
 180         need_set = set()
 181         bugzilla = None
 182         for nodes in budget_graph.assigned_nodes_for_milestones.values():
 183             for node in nodes:
 184                 need_set.add(node.bug.id)
 185                 bugzilla = node.bug.bugzilla
 186         need_list = sorted(need_set)
 187         total = len(need_list)
 188         with tty_out() as term:
 189             step = 100
 190             i = 0
 191             while i < total:
 192                 cur_need = need_list[i:i + step]
 193                 stop = i + len(cur_need)
 194                 print("loading comments %d:%d of %d" % (i, stop, total),
 195                       flush=True, file=term)
 196                 comments = bugzilla.get_comments(cur_need)['bugs']
 197                 if len(comments) < len(cur_need) and len(cur_need) > 1:
 198                     step = max(1, step // 2)
 199                     print("failed, trying smaller step of %d" % step,
 200                           flush=True, file=term)
 201                     continue
 202                 bug_comments_map.update(comments)
 203                 i += len(cur_need)
 204     for milestone, payments in budget_graph.milestone_payments.items():
 205         summary = PaymentSummary(payments)
 206         # and one to display people
 207         ppl = []
 208         for person in budget_graph.milestone_people[milestone]:
 209             p = {'name': person.full_name, 'email': person.email}
 210             ppl.append(p)
 211
 212         tasks = []
 213         canonical = budget_graph.nodes[milestone.canonical_bug_id]
 214         for child in canonical.immediate_children:
 215             milestones = []
 216             # include the task itself as a milestone
 217             for st in list(child.children()) + [child]:
 218                 amount = st.fixed_budget_excluding_subtasks.int()
 219                 if amount == 0:  # skip anything at zero
 220                     continue
 221                 # if "task itself" then put the milestone as "wrapup"
 222                 if st.bug == child.bug:
 223                     description = 'wrapup'
 224                     intro = []
 225                 else:
 226                     # otherwise create a description and get comment #0
 227                     description = "%d %s" % (st.bug.id, st.bug.summary)
 228                     # add parent and MoU top-level
 229                     parent_id = st.parent.bug.id
 230                     if parent_id != child.bug.id:
 231                         description += "\n(Sub-sub-task of %d)" % parent_id
 232                 task = {'description': description,
 233                         'amount': amount,
 234                         }
 235                 #mou_bug = st.closest_bug_in_mou
 236                 # if mou_bug is not None:
 237                 #    task['mou_task'] = mou_bug.bug.id
 238                 milestones.append(task)
 239             # create MoU task: get comment #0
 240             intro = []
 241             comment = "%s\n " % child.bug_url
 242             if add_comments:
 243                 comment += "\n"
 244                 comments = bug_comments_map[str(child.bug.id)]['comments']
 245                 lines = comments[0]['text'].splitlines()
 246                 for i, line in enumerate(lines):
 247                     # look for a line with only 2 or more `-` as the
 248                     # standard way in markdown of having a "break" (<hl />)
 249                     # this truncates the comment so that the RFP database
 250                     # has only the "summary description" but the rest may
 251                     # be used for "TODO" lists
 252                     l = line.strip()
 253                     if len(l) >= 2 and l == "-" * len(l):
 254                         lines[i:] = []
 255                         break
 256                 comment += "\n".join(lines)
 257             intro.append(comment)
 258             # print (description, intro)
 259             # sys.stdout.flush()
 260             task = {'title': "%d %s" % (child.bug.id, child.bug.summary),
 261                     'intro': intro,
 262                     'amount': child.fixed_budget_including_subtasks.int(),
 263                     'url': "{{ %s }} " % child.bug_url,
 264                     'milestones': milestones
 265                     }
 266             tasks.append(task)
 267
 268         d = {'participants': ppl,
 269              'preamble': '',
 270              'type': 'Group',
 271              'url': canonical.bug_url,
 272              'plan': {'intro': [''],
 273                       'tasks': tasks,
 274                       'rfp_secret': '',
 275                       }
 276              }
 277
 278         output_file = output_dir / f"report.{milestone.identifier}.json"
 279         output_file.write_text(json.dumps(d, indent=2), encoding="utf-8")
 280
 281
 282 if __name__ == "__main__":
 283     main()