diff --git a/.gitignore b/.gitignore index 0015062..4ca041c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,7 @@ /secure/ .idea -/logs/* -!/logs/.gitkeep +*/logs/* Downloads *.csv diff --git a/SngFile.py b/SngFile.py index 57e661c..5e7137b 100644 --- a/SngFile.py +++ b/SngFile.py @@ -1,21 +1,15 @@ """This file is used to define SngFile class and somee helper methods related to it's usage.""" -import json import logging import logging.config import re from itertools import chain -from pathlib import Path import SNG_DEFAULTS from sng_utils import generate_verse_marker_from_line, validate_suspicious_encoding_str from SngFileHeaderValidationPart import SngFileHeaderValidation from SngFileParserPart import SngFileParserPart -config_file = Path("logging_config.json") -with config_file.open(encoding="utf-8") as f_in: - logging_config = json.load(f_in) - logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/logs/.gitkeep b/logs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/main.py b/main.py index d64db88..7112c27 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,6 @@ It mainly works based on df comparison """ -import json import logging import logging.config import os.path @@ -702,40 +701,3 @@ def validate_ct_song_sng_count(api: ChurchToolsApi) -> None: # avoid Too many requests. Rate Limit Exceeded. logger.debug("sleep 1 second after %s / %s", song_count, len_songs) time.sleep(1) - - -if __name__ == "__main__": - config_file = Path("logging_config.json") - with config_file.open(encoding="utf-8") as f_in: - logging_config = json.load(f_in) - logging.config.dictConfig(config=logging_config) - logger.info("Excecuting Main RUN") - - songs_temp = [] - df_sng = read_songs_to_df() - df_sng = clean_all_songs(df_sng=df_sng) - write_df_to_file(df_sng) - - api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token) - validate_ct_song_sng_count(api) - - # Match all SongIDs from CT to local songs where missing - df_ct = get_ct_songs_as_df(api) - add_id_to_local_song_if_available_in_ct(df_sng, df_ct) - - # Upload all songs into CT that are new - df_ct = get_ct_songs_as_df(api) - upload_new_local_songs_and_generate_ct_id(df_sng, df_ct) - - # To be safe - re-read all data sources and upload - df_sng = read_songs_to_df() - df_ct = get_ct_songs_as_df(api) - download_missing_online_songs(df_sng, df_ct, api) - - """ - df_sng = read_baiersbronn_songs_to_df() - df_ct = get_ct_songs_as_df() - upload_local_songs_by_id(df_sng, df_ct) - """ - - logger.info("Main Method finished") diff --git a/notebooks/qs_ct_only.ipynb b/notebooks/qs_ct_only.ipynb new file mode 100644 index 0000000..7f06799 --- /dev/null +++ b/notebooks/qs_ct_only.ipynb @@ -0,0 +1,116 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import logging\n", + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "from ChurchToolsApi import ChurchToolsApi\n", + "\n", + "parent_path = str(Path().absolute().parent)\n", + "if parent_path not in sys.path:\n", + " print(f\"adding {parent_path} to parent path\")\n", + " sys.path.insert(1, parent_path)\n", + "\n", + "from main import validate_ct_song_sng_count\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "logger = logging.getLogger(__name__)\n", + "\n", + "config_file = Path(\"../logging_config.json\")\n", + "with config_file.open(encoding=\"utf-8\") as f_in:\n", + " logging_config = json.load(f_in)\n", + " log_directory = Path(logging_config[\"handlers\"][\"file\"][\"filename\"]).parent\n", + " if not log_directory.exists():\n", + " log_directory.mkdir(parents=True)\n", + " logging.config.dictConfig(config=logging_config)\n", + "logger.info(\"Excecuting Main RUN\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ct_domain = os.getenv(\"CT_DOMAIN\")\n", + "ct_token = os.getenv(\"CT_TOKEN\")\n", + "\n", + "if ct_domain is None or ct_token is None:\n", + " from secure.config import ct_domain, ct_token\n", + "\n", + " logger.info(\n", + " \"ct_domain or ct_token missing in env variables - using local config instead\"\n", + " )\n", + " from secure import config\n", + "\n", + " ct_domain = config.ct_domain\n", + " ct_token = config.ct_token\n", + "\n", + "api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token)\n", + "\n", + "connected_as = api.who_am_i()\n", + "\n", + "print(f\"This notebook will connect to {ct_domain} as {connected_as['firstName']} {connected_as['lastName']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tags" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#CT validation\n", + "validate_ct_song_sng_count(api)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/update.ipynb b/notebooks/update.ipynb new file mode 100644 index 0000000..771d370 --- /dev/null +++ b/notebooks/update.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import logging\n", + "import os\n", + "import sys\n", + "from pathlib import Path\n", + "\n", + "from ChurchToolsApi import ChurchToolsApi\n", + "\n", + "parent_path = str(Path().absolute().parent)\n", + "if parent_path not in sys.path:\n", + " print(f\"adding {parent_path} to parent path\")\n", + " sys.path.insert(1, parent_path)\n", + "\n", + "from main import (\n", + " add_id_to_local_song_if_available_in_ct,\n", + " clean_all_songs,\n", + " download_missing_online_songs,\n", + " get_ct_songs_as_df,\n", + " read_songs_to_df,\n", + " upload_new_local_songs_and_generate_ct_id,\n", + " write_df_to_file,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "logger = logging.getLogger(__name__)\n", + "\n", + "config_file = Path(\"../logging_config.json\")\n", + "with config_file.open(encoding=\"utf-8\") as f_in:\n", + " logging_config = json.load(f_in)\n", + " log_directory = Path(logging_config[\"handlers\"][\"file\"][\"filename\"]).parent\n", + " if not log_directory.exists():\n", + " log_directory.mkdir(parents=True)\n", + " logging.config.dictConfig(config=logging_config)\n", + "logger.info(\"Excecuting Main RUN\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ct_domain = os.getenv(\"CT_DOMAIN\")\n", + "ct_token = os.getenv(\"CT_TOKEN\")\n", + "\n", + "if ct_domain is None or ct_token is None:\n", + " from secure.config import ct_domain, ct_token\n", + "\n", + " logger.info(\n", + " \"ct_domain or ct_token missing in env variables - using local config instead\"\n", + " )\n", + " from secure import config\n", + "\n", + " ct_domain = config.ct_domain\n", + " ct_token = config.ct_token\n", + "\n", + "api = ChurchToolsApi(domain=ct_domain, ct_token=ct_token)\n", + "\n", + "connected_as = api.who_am_i()\n", + "\n", + "print(f\"This notebook will connect to {ct_domain} as {connected_as['firstName']} {connected_as['lastName']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Read local SNG files\n", + "* read songs\n", + "* clean songs\n", + "* overwrite songs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "songs_temp = []\n", + "df_sng = read_songs_to_df()\n", + "df_sng = clean_all_songs(df_sng=df_sng)\n", + "write_df_to_file(df_sng)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get CT songs \n", + "* update local IDs\n", + "* upload new local songs\n", + "* download missing online songs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Match all SongIDs from CT to local songs where missing\n", + "df_ct = get_ct_songs_as_df(api)\n", + "add_id_to_local_song_if_available_in_ct(df_sng, df_ct)\n", + "\n", + "# Upload all songs into CT that are new\n", + "df_ct = get_ct_songs_as_df(api)\n", + "upload_new_local_songs_and_generate_ct_id(df_sng, df_ct)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "download_missing_online_songs(df_sng, df_ct, api)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\"\"\"\n", + "df_sng = read_baiersbronn_songs_to_df()\n", + "df_ct = get_ct_songs_as_df()\n", + "upload_local_songs_by_id(df_sng, df_ct)\n", + "\"\"\"\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ruff.toml b/ruff.toml index be6105f..3b68082 100644 --- a/ruff.toml +++ b/ruff.toml @@ -44,9 +44,14 @@ ignore = [ #Abstacts used to split file "B024", #is an abstract base class, but it has no abstract methods ] - extend-ignore = [] #Skip rules that need more code cleaning... +[lint.per-file-ignores] +"notebooks/*.ipynb" = [ + "T201", #print + "E402" #import outside top-level + ] + [lint.pydocstyle] convention = "google" diff --git a/sng_utils.py b/sng_utils.py index 83905ee..ae9d8e5 100644 --- a/sng_utils.py +++ b/sng_utils.py @@ -1,17 +1,11 @@ """This module includes utilities used independant of sng instances.""" -import json import logging import logging.config import re -from pathlib import Path import SNG_DEFAULTS -config_file = Path("logging_config.json") -with config_file.open(encoding="utf-8") as f_in: - logging_config = json.load(f_in) - logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/tests/test_main.py b/tests/test_main.py index 83d701f..5f8797d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -36,6 +36,9 @@ config_file = Path("logging_config.json") with config_file.open(encoding="utf-8") as f_in: logging_config = json.load(f_in) + log_directory = Path(logging_config["handlers"]["file"]["filename"]).parent + if not log_directory.exists(): + log_directory.mkdir(parents=True) logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/tests/test_sng.py b/tests/test_sng.py index 32fb240..3ec233c 100644 --- a/tests/test_sng.py +++ b/tests/test_sng.py @@ -11,6 +11,9 @@ config_file = Path("logging_config.json") with config_file.open(encoding="utf-8") as f_in: logging_config = json.load(f_in) + log_directory = Path(logging_config["handlers"]["file"]["filename"]).parent + if not log_directory.exists(): + log_directory.mkdir(parents=True) logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/tests/test_sng_header_validation.py b/tests/test_sng_header_validation.py index 5021a28..9596777 100644 --- a/tests/test_sng_header_validation.py +++ b/tests/test_sng_header_validation.py @@ -13,6 +13,9 @@ config_file = Path("logging_config.json") with config_file.open(encoding="utf-8") as f_in: logging_config = json.load(f_in) + log_directory = Path(logging_config["handlers"]["file"]["filename"]).parent + if not log_directory.exists(): + log_directory.mkdir(parents=True) logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/tests/test_sng_parser.py b/tests/test_sng_parser.py index a1cd0ad..406f5bd 100644 --- a/tests/test_sng_parser.py +++ b/tests/test_sng_parser.py @@ -13,6 +13,9 @@ config_file = Path("logging_config.json") with config_file.open(encoding="utf-8") as f_in: logging_config = json.load(f_in) + log_directory = Path(logging_config["handlers"]["file"]["filename"]).parent + if not log_directory.exists(): + log_directory.mkdir(parents=True) logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__) diff --git a/tests/test_sng_utils.py b/tests/test_sng_utils.py index 14125b2..7ea5146 100644 --- a/tests/test_sng_utils.py +++ b/tests/test_sng_utils.py @@ -11,6 +11,9 @@ config_file = Path("logging_config.json") with config_file.open(encoding="utf-8") as f_in: logging_config = json.load(f_in) + log_directory = Path(logging_config["handlers"]["file"]["filename"]).parent + if not log_directory.exists(): + log_directory.mkdir(parents=True) logging.config.dictConfig(config=logging_config) logger = logging.getLogger(__name__)