main to jupyter notebook and logging update #22

* Create a Jupyter Notebook instead of main.py __init__ - Fixes #22 * updated logging config including dir creation #22 * ignoring logs in subdirs too #22 * adding option to create log directory to tests #22
bensteUEM · May 9, 2024 · b9ad53d · b9ad53d
1 parent e3396f9
commit b9ad53d
Show file tree

Hide file tree

Showing 13 changed files with 315 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,8 +3,7 @@
 
 /secure/
 .idea
-/logs/*
-!/logs/.gitkeep
+*/logs/*
 Downloads
 *.csv
 

diff --git a/SngFile.py b/SngFile.py
@@ -1,21 +1,15 @@
 """This file is used to define SngFile class and somee helper methods related to it's usage."""
 
-import json
 import logging
 import logging.config
 import re
 from itertools import chain
-from pathlib import Path
 
 import SNG_DEFAULTS
 from sng_utils import generate_verse_marker_from_line, validate_suspicious_encoding_str
 from SngFileHeaderValidationPart import SngFileHeaderValidation
 from SngFileParserPart import SngFileParserPart
 
-config_file = Path("logging_config.json")
-with config_file.open(encoding="utf-8") as f_in:
-    logging_config = json.load(f_in)
-    logging.config.dictConfig(config=logging_config)
 logger = logging.getLogger(__name__)
 
 

diff --git a/logs/.gitkeep b/logs/.gitkeep
diff --git a/main.py b/main.py
@@ -4,7 +4,6 @@
 It mainly works based on df comparison
 """
 
-import json
 import logging
 import logging.config
 import os.path
@@ -702,40 +701,3 @@ def validate_ct_song_sng_count(api: ChurchToolsApi) -> None:
             # avoid Too many requests. Rate Limit Exceeded.
             logger.debug("sleep 1 second after %s / %s", song_count, len_songs)
             time.sleep(1)
-
-
-if __name__ == "__main__":
-    config_file = Path("logging_config.json")
-    with config_file.open(encoding="utf-8") as f_in:
-        logging_config = json.load(f_in)
-        logging.config.dictConfig(config=logging_config)
-    logger.info("Excecuting Main RUN")
-
-    songs_temp = []
-    df_sng = read_songs_to_df()
-    df_sng = clean_all_songs(df_sng=df_sng)
-    write_df_to_file(df_sng)
-
-    api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token)
-    validate_ct_song_sng_count(api)
-
-    # Match all SongIDs from CT to local songs where missing
-    df_ct = get_ct_songs_as_df(api)
-    add_id_to_local_song_if_available_in_ct(df_sng, df_ct)
-
-    # Upload all songs into CT that are new
-    df_ct = get_ct_songs_as_df(api)
-    upload_new_local_songs_and_generate_ct_id(df_sng, df_ct)
-
-    # To be safe - re-read all data sources and upload
-    df_sng = read_songs_to_df()
-    df_ct = get_ct_songs_as_df(api)
-    download_missing_online_songs(df_sng, df_ct, api)
-
-    """
-    df_sng = read_baiersbronn_songs_to_df()
-    df_ct = get_ct_songs_as_df()
-    upload_local_songs_by_id(df_sng, df_ct)
-    """
-
-    logger.info("Main Method finished")
diff --git a/notebooks/qs_ct_only.ipynb b/notebooks/qs_ct_only.ipynb
@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import logging\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "from ChurchToolsApi import ChurchToolsApi\n",
+    "\n",
+    "parent_path = str(Path().absolute().parent)\n",
+    "if parent_path not in sys.path:\n",
+    "    print(f\"adding {parent_path} to parent path\")\n",
+    "    sys.path.insert(1, parent_path)\n",
+    "\n",
+    "from main import validate_ct_song_sng_count\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "logger = logging.getLogger(__name__)\n",
+    "\n",
+    "config_file = Path(\"../logging_config.json\")\n",
+    "with config_file.open(encoding=\"utf-8\") as f_in:\n",
+    "    logging_config = json.load(f_in)\n",
+    "    log_directory = Path(logging_config[\"handlers\"][\"file\"][\"filename\"]).parent\n",
+    "    if not log_directory.exists():\n",
+    "        log_directory.mkdir(parents=True)\n",
+    "    logging.config.dictConfig(config=logging_config)\n",
+    "logger.info(\"Excecuting Main RUN\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ct_domain = os.getenv(\"CT_DOMAIN\")\n",
+    "ct_token = os.getenv(\"CT_TOKEN\")\n",
+    "\n",
+    "if ct_domain is None or ct_token is None:\n",
+    "    from secure.config import ct_domain, ct_token\n",
+    "\n",
+    "    logger.info(\n",
+    "        \"ct_domain or ct_token missing in env variables - using local config instead\"\n",
+    "    )\n",
+    "    from secure import config\n",
+    "\n",
+    "    ct_domain = config.ct_domain\n",
+    "    ct_token = config.ct_token\n",
+    "\n",
+    "api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token)\n",
+    "\n",
+    "connected_as = api.who_am_i()\n",
+    "\n",
+    "print(f\"This notebook will connect to {ct_domain} as {connected_as['firstName']} {connected_as['lastName']}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tags"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#CT validation\n",
+    "validate_ct_song_sng_count(api)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/update.ipynb b/notebooks/update.ipynb
@@ -0,0 +1,177 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import logging\n",
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "from ChurchToolsApi import ChurchToolsApi\n",
+    "\n",
+    "parent_path = str(Path().absolute().parent)\n",
+    "if parent_path not in sys.path:\n",
+    "    print(f\"adding {parent_path} to parent path\")\n",
+    "    sys.path.insert(1, parent_path)\n",
+    "\n",
+    "from main import (\n",
+    "    add_id_to_local_song_if_available_in_ct,\n",
+    "    clean_all_songs,\n",
+    "    download_missing_online_songs,\n",
+    "    get_ct_songs_as_df,\n",
+    "    read_songs_to_df,\n",
+    "    upload_new_local_songs_and_generate_ct_id,\n",
+    "    write_df_to_file,\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "logger = logging.getLogger(__name__)\n",
+    "\n",
+    "config_file = Path(\"../logging_config.json\")\n",
+    "with config_file.open(encoding=\"utf-8\") as f_in:\n",
+    "    logging_config = json.load(f_in)\n",
+    "    log_directory = Path(logging_config[\"handlers\"][\"file\"][\"filename\"]).parent\n",
+    "    if not log_directory.exists():\n",
+    "        log_directory.mkdir(parents=True)\n",
+    "    logging.config.dictConfig(config=logging_config)\n",
+    "logger.info(\"Excecuting Main RUN\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ct_domain = os.getenv(\"CT_DOMAIN\")\n",
+    "ct_token = os.getenv(\"CT_TOKEN\")\n",
+    "\n",
+    "if ct_domain is None or ct_token is None:\n",
+    "    from secure.config import ct_domain, ct_token\n",
+    "\n",
+    "    logger.info(\n",
+    "        \"ct_domain or ct_token missing in env variables - using local config instead\"\n",
+    "    )\n",
+    "    from secure import config\n",
+    "\n",
+    "    ct_domain = config.ct_domain\n",
+    "    ct_token = config.ct_token\n",
+    "\n",
+    "api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token)\n",
+    "\n",
+    "connected_as = api.who_am_i()\n",
+    "\n",
+    "print(f\"This notebook will connect to {ct_domain} as {connected_as['firstName']} {connected_as['lastName']}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Read local SNG files\n",
+    "* read songs\n",
+    "* clean songs\n",
+    "* overwrite songs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "songs_temp = []\n",
+    "df_sng = read_songs_to_df()\n",
+    "df_sng = clean_all_songs(df_sng=df_sng)\n",
+    "write_df_to_file(df_sng)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get CT songs \n",
+    "* update local IDs\n",
+    "* upload new local songs\n",
+    "* download missing online songs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Match all SongIDs from CT to local songs where missing\n",
+    "df_ct = get_ct_songs_as_df(api)\n",
+    "add_id_to_local_song_if_available_in_ct(df_sng, df_ct)\n",
+    "\n",
+    "# Upload all songs into CT that are new\n",
+    "df_ct = get_ct_songs_as_df(api)\n",
+    "upload_new_local_songs_and_generate_ct_id(df_sng, df_ct)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "download_missing_online_songs(df_sng, df_ct, api)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "\"\"\"\n",
+    "df_sng = read_baiersbronn_songs_to_df()\n",
+    "df_ct = get_ct_songs_as_df()\n",
+    "upload_local_songs_by_id(df_sng, df_ct)\n",
+    "\"\"\"\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/ruff.toml b/ruff.toml
@@ -44,9 +44,14 @@ ignore = [
     #Abstacts used to split file
     "B024", #is an abstract base class, but it has no abstract methods
 ]
-
 extend-ignore = [] #Skip rules that need more code cleaning...
 
+[lint.per-file-ignores]
+"notebooks/*.ipynb" = [
+    "T201", #print
+    "E402" #import outside top-level
+    ]
+
 [lint.pydocstyle]
 convention = "google"