generate_gource.py: support multiple projects

Add support of downloading and combining histories of several JIRA
projects.  Gource log format operates on filepaths.  For the
visualization to be informative, group tickets by project in Gource log.

history_converter.py:
  To make the visualization grouped by JIRA project, teach method
  convert_history to group files in Gource log into top level directory
  named by ID of a project: teach method get_filename to prepend paths
  with the project's ID.

jira.py:
  Introduce method download_projects.  Move loading and saving of cache
  of downloaded JIRA tickets on module level.

README.md

@@ -116,13 +116,6 @@ TODO
 
 * Download avatars directly from JIRA server using URL of the following form
   `<JIRA-SERVER>/jira/secure/useravatar?size=large&ownerId=fred`
-* Add ability to download several projects at once, and a way to combine
-  several histories into one, perhaps grouping them in folders by project key
-  (or a more detailed project name, see
-  [GET project](https://docs.atlassian.com/software/jira/docs/api/REST/7.6.1/#api/2/project-getProject)),
-  to complement the existing partition by summary prefixes (see function
-  `generate_folder()` in `history_converter.py`) and custom folder (see
-  description of `sections_extension` in "Input" section).
 * Flip boolean logic from `skip_filter` to `changelog_predicate`, to correspond
   to the first parameter of Python's `filter` builtin.
 * Retire `skip_dates`, as `skip_filter` is much more versatile.  In the sample

generate_gource.py

@@ -18,12 +18,17 @@ def to_str(t) -> str:
     return _create_gource(*t)
 
 
-project_id = list(config.projects.keys())[0]
-changes, tickets_json = jira.download_project(project_id)
-gource_list = convert_history(tickets_json, changes,
-                              config.projects[project_id].get('sections_extension', None))
-
-gource_input_txt = "gource-input-{0}.txt".format(project_id)
+changes, tickets_json = jira.download_projects(list(config.projects.keys()))
+gource_list = []
+for project_id in config.projects:
+    project_changes = changes[project_id]
+    project_gource_list = convert_history(tickets_json,
+                                          project_id, project_changes,
+                                          config.projects[project_id].get('sections_extension', None))
+    gource_list.extend(project_gource_list)
+gource_list = sorted(gource_list)
+
+gource_input_txt = 'gource-input.txt'
 try:
     start = current_milli_time()
     with open(gource_input_txt, "w", encoding='utf-8') as gource_file:
@@ -38,3 +43,5 @@ def to_str(t) -> str:
 except Exception as e:
     print("Unexpected exception", e)
     print("Bailing out")
+finally:
+    jira.save_cache()

history_converter.py

@@ -21,21 +21,19 @@ def generate_extension(tickets_json, jira_key: str) -> str:
     return "." + issuetype.replace(" ", "")
 
 
-def generate_folder(tickets_json, jira_key: str, extension) -> str:
+def generate_folder(tickets_json, project_id: str, jira_key: str, extension) -> str:
     summary = jira.get_issue_json(tickets_json, jira_key)['fields']['summary']
     sections = list(map(lambda s: s.strip().title(), summary.split(':')))
     sections = sections[:-1]  # remove last bit of summary
     sections = list(filter(lambda s: len(s) < 50, sections))
     if extension is not None:
         sections = extension(jira.get_issue_json(tickets_json, jira_key), sections)
-    # TODO (several projects) replace two lines with
-    # TODO sections.insert(0, project_id)
-    if len(sections) == 0:
-        return ""
+    sections.insert(0, project_id)
     return '/'.join(sections) + '/'
 
 
 def convert_history(tickets_json,
+                    project_id: str,
                     modifications: List[Tuple[int, str, str, bool]],
                     sections_extension):
     print("Number of changes: ", len(modifications))
@@ -48,7 +46,7 @@ def convert_history(tickets_json,
 
     @lru_cache(maxsize=50000)
     def get_filename(jira_key: str) -> str:
-        folder_path = generate_folder(tickets_json, jira_key, sections_extension)
+        folder_path = generate_folder(tickets_json, project_id, jira_key, sections_extension)
         return folder_path + jira_key + generate_extension(tickets_json, jira_key)
 
     try:

jira.py

@@ -149,6 +149,13 @@ def is_good_date(bad_dates, changelog_entry):
 print("Missing tickets count = {}".format(len(missing_tickets)))
 if JIRA_DEBUG:
     print("Missing tickets: ", ", ".join(sorted(missing_tickets)))
+
+# Loading tickets cache
+tickets_title = 'tickets'
+tickets_json = load_json(tickets_title)
+if tickets_json is None:
+    tickets_json = {}
+
 rest_session = requests.Session()
 
 default_fields = {'key', 'summary', 'issuetype'}
@@ -186,10 +193,6 @@ def entry_predicate(changelog_entry):
     max_key = project_config['max_key']
     extra_fields = project_config['extra_fields'] or ''
     fields = ','.join(default_fields.union(set(extra_fields)))
-    tickets_title = project_id + '-tickets'
-    tickets_json = load_json(tickets_title)
-    if tickets_json is None:
-        tickets_json = {}
 
     # Download of tickets
     for i in range(min_key, max_key):
@@ -222,10 +225,6 @@ def entry_predicate(changelog_entry):
             continue
         _put_history(tickets_json, key, filtered_history(tickets_json, key, entry_predicate))
 
-    # store all the tickets
-    print("Total number of tickets: {0}".format(len(tickets_json)))
-    save_json(tickets_title, tickets_json)
-
     tickets_to_process = []
     for i in range(min_key, max_key):
         key = get_key_str(project_id, i)
@@ -238,13 +237,24 @@ def entry_predicate(changelog_entry):
         tickets_to_process.append(key)
 
     project_changes = []
-    projects[project_id] = (project_changes, tickets_json)
+    projects[project_id] = project_changes
     for key in tickets_to_process:
         history = _pop_history(tickets_json, key)
         project_changes.extend(history)
+    return projects[project_id]
 
+
+def download_projects(project_ids: List[str]):
+    for project_id in project_ids:
+        download_project(project_id)
+    return projects, tickets_json
+
+
+def save_cache():
+    # store all the tickets
+    print("Total number of tickets: {0}".format(len(tickets_json)))
+    save_json(tickets_title, tickets_json)
     print("Saving " + missing_file_path)
     with open(missing_file_path, "w") as f:
         f.write("\n".join(sorted(missing_tickets)))
     print("Saved!")
-    return projects[project_id]