Merge pull request #1 from os-climate/osc-ingest-tools-0-5-1

Update to osc-ingest-tools 0.5.1
os-climate · Jun 5, 2024 · 9b65fd4 · 9b65fd4
2 parents 97447d9 + 5439f74
commit 9b65fd4
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 50 deletions.
diff --git a/notebooks/iso3166-ingest.ipynb b/notebooks/iso3166-ingest.ipynb
@@ -1,13 +1,5 @@
 {
  "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7dec07e8-1c7d-4864-8d7f-7b8079fc8aef",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "id": "2e9358c2-843e-4c9f-8a72-01c1b46c82a7",
@@ -57,7 +49,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 1,
    "id": "3b5ed789-9a67-400f-b08d-bffb6e48e1af",
    "metadata": {},
    "outputs": [],
@@ -68,7 +60,8 @@
     "import pathlib\n",
     "\n",
     "import trino\n",
-    "from sqlalchemy.engine import create_engine"
+    "from sqlalchemy.engine import create_engine\n",
+    "from sqlalchemy import text"
    ]
   },
   {
@@ -83,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 2,
    "id": "36002a98-522f-43eb-91d8-75e92305f41c",
    "metadata": {},
    "outputs": [],
@@ -101,7 +94,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 3,
    "id": "6fb8aedc-1d0b-4aff-9fce-901fde5c10d3",
    "metadata": {},
    "outputs": [],
@@ -121,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 4,
    "id": "fb3678ba-ab2e-4d1e-b812-636315f0fc8c",
    "metadata": {},
    "outputs": [],
@@ -132,12 +125,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 5,
    "id": "3fa6bc63-4a78-4a4f-81a5-872ca3eb6577",
    "metadata": {},
    "outputs": [],
    "source": [
-    "trino_bucket = attach_s3_bucket(\"S3_DEV\")"
+    "trino_bucket = osc.attach_s3_bucket(\"S3_DEV\")"
    ]
   },
   {
@@ -150,7 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 6,
    "id": "75e44b1a-b3fc-4de7-a8a3-0b7b5c806cd2",
    "metadata": {},
    "outputs": [
@@ -171,7 +164,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 7,
    "id": "cf7ed364-7b63-4f85-9763-12da29ca7281",
    "metadata": {},
    "outputs": [
@@ -225,7 +218,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "id": "f55ffe79-61ea-45f4-9b44-eccf0a74e879",
    "metadata": {
     "tags": []
@@ -254,7 +247,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "id": "40ccf11d-1694-46be-ab3b-61c991e259b5",
    "metadata": {
     "tags": []
@@ -265,8 +258,8 @@
      "output_type": "stream",
      "text": [
       "show tables in mdt_sandbox\n",
-      "[('assets_earnings_investments',), ('assets_usd_by_lei',), ('assets_xyz_by_lei',), ('average_fx',), ('cash_usd_by_lei',), ('cash_xyz_by_lei',), ('closing_fx',), ('customers_sales',), ('debt_equity_returns',), ('debt_usd_by_lei',), ('debt_xyz_by_lei',), ('emissions_targets',), ('employees',), ('expenditure_bills_burden',), ('financials_by_lei',), ('float_usd_by_lei',), ('float_xyz_by_lei',), ('fy_income_usd_by_lei',), ('fy_income_xyz_by_lei',), ('fy_revenue_usd_by_lei',), ('fy_revenue_xyz_by_lei',), ('housing_units_income',), ('itr_portfolio_universe',), ('net_plant_balance',), ('operations_emissions_by_fuel',), ('operations_emissions_by_tech',), ('portfolio_universe',), ('revenue_by_tech',), ('rmi_assets_earnings_investments',), ('rmi_assets_earnings_investments_source',), ('rmi_customers_sales',), ('rmi_customers_sales_source',), ('rmi_debt_equity_returns',), ('rmi_debt_equity_returns_source',), ('rmi_emissions_targets',), ('rmi_emissions_targets_source',), ('rmi_employees',), ('rmi_employees_source',), ('rmi_expenditure_bills_burden',), ('rmi_expenditure_bills_burden_source',), ('rmi_housing_units_income',), ('rmi_housing_units_income_source',), ('rmi_net_plant_balance',), ('rmi_net_plant_balance_source',), ('rmi_operations_emissions_by_fuel',), ('rmi_operations_emissions_by_fuel_source',), ('rmi_operations_emissions_by_tech',), ('rmi_operations_emissions_by_tech_source',), ('rmi_revenue_by_tech',), ('rmi_revenue_by_tech_source',), ('rmi_state_policies',), ('rmi_state_policies_source',), ('rmi_state_targets',), ('rmi_state_targets_source',), ('rmi_utility_information',), ('rmi_utility_information_2023',), ('rmi_utility_information_2023_source',), ('rmi_utility_information_source',), ('rmi_utility_state_map',), ('rmi_utility_state_map_2023',), ('rmi_utility_state_map_2023_source',), ('rmi_utility_state_map_source',), ('saved_queries',), ('sf_oecd_imgr_fco2',), ('sf_wdi_gdp',), ('state_policies',), ('state_targets',), ('utility_information',), ('utility_information_2023',), ('utility_state_map',), ('utility_state_map_2023',)]\n",
-      "[('assets_earnings_investments',), ('assets_usd_by_lei',), ('assets_xyz_by_lei',), ('average_fx',), ('cash_usd_by_lei',), ('cash_xyz_by_lei',), ('closing_fx',), ('customers_sales',), ('debt_equity_returns',), ('debt_usd_by_lei',), ('debt_xyz_by_lei',), ('emissions_targets',), ('employees',), ('expenditure_bills_burden',), ('financials_by_lei',), ('float_usd_by_lei',), ('float_xyz_by_lei',), ('fy_income_usd_by_lei',), ('fy_income_xyz_by_lei',), ('fy_revenue_usd_by_lei',), ('fy_revenue_xyz_by_lei',), ('housing_units_income',), ('itr_portfolio_universe',), ('net_plant_balance',), ('operations_emissions_by_fuel',), ('operations_emissions_by_tech',), ('portfolio_universe',), ('revenue_by_tech',), ('rmi_assets_earnings_investments',), ('rmi_assets_earnings_investments_source',), ('rmi_customers_sales',), ('rmi_customers_sales_source',), ('rmi_debt_equity_returns',), ('rmi_debt_equity_returns_source',), ('rmi_emissions_targets',), ('rmi_emissions_targets_source',), ('rmi_employees',), ('rmi_employees_source',), ('rmi_expenditure_bills_burden',), ('rmi_expenditure_bills_burden_source',), ('rmi_housing_units_income',), ('rmi_housing_units_income_source',), ('rmi_net_plant_balance',), ('rmi_net_plant_balance_source',), ('rmi_operations_emissions_by_fuel',), ('rmi_operations_emissions_by_fuel_source',), ('rmi_operations_emissions_by_tech',), ('rmi_operations_emissions_by_tech_source',), ('rmi_revenue_by_tech',), ('rmi_revenue_by_tech_source',), ('rmi_state_policies',), ('rmi_state_policies_source',), ('rmi_state_targets',), ('rmi_state_targets_source',), ('rmi_utility_information',), ('rmi_utility_information_2023',), ('rmi_utility_information_2023_source',), ('rmi_utility_information_source',), ('rmi_utility_state_map',), ('rmi_utility_state_map_2023',), ('rmi_utility_state_map_2023_source',), ('rmi_utility_state_map_source',), ('saved_queries',), ('sf_oecd_imgr_fco2',), ('sf_wdi_gdp',), ('state_policies',), ('state_targets',), ('utility_information',), ('utility_information_2023',), ('utility_state_map',), ('utility_state_map_2023',)]\n"
+      "[('isic_to_sector',), ('itr_portfolio_universe',), ('pcaf_dbt_models',), ('portfolio_universe',), ('saved_queries',), ('sf_oecd_exch_rates',), ('sf_oecd_exch_rates_source',), ('sf_oecd_exgr_dco2',), ('sf_oecd_exgr_dco2_source',), ('sf_oecd_imgr_fco2',), ('sf_oecd_imgr_fco2_source',), ('sf_primap_hist_emissions',), ('sf_primap_hist_emissions_source',), ('sf_total_sovereign_emissions',), ('sf_total_sovereign_emissions_source',), ('sf_unfccc_countries_source',), ('sf_unfccc_results',), ('sf_unfccc_results_source',), ('sf_unfccc_with_lulucf',), ('sf_unfccc_with_lulucf_source',), ('sf_unfccc_without_lulucf',), ('sf_unfccc_without_lulucf_source',), ('sf_wdi_gdp',), ('sf_wdi_gdp_source',), ('sf_wdi_population',), ('sf_wdi_population_source',)]\n",
+      "[('isic_to_sector',), ('itr_portfolio_universe',), ('pcaf_dbt_models',), ('portfolio_universe',), ('saved_queries',), ('sf_oecd_exch_rates',), ('sf_oecd_exch_rates_source',), ('sf_oecd_exgr_dco2',), ('sf_oecd_exgr_dco2_source',), ('sf_oecd_imgr_fco2',), ('sf_oecd_imgr_fco2_source',), ('sf_primap_hist_emissions',), ('sf_primap_hist_emissions_source',), ('sf_total_sovereign_emissions',), ('sf_total_sovereign_emissions_source',), ('sf_unfccc_countries_source',), ('sf_unfccc_results',), ('sf_unfccc_results_source',), ('sf_unfccc_with_lulucf',), ('sf_unfccc_with_lulucf_source',), ('sf_unfccc_without_lulucf',), ('sf_unfccc_without_lulucf_source',), ('sf_wdi_gdp',), ('sf_wdi_gdp_source',), ('sf_wdi_population',), ('sf_wdi_population_source',)]\n"
      ]
     }
    ],
@@ -305,32 +298,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 10,
    "id": "711a5bf3-5299-4959-9c68-b5169507c2f4",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[('AFG', 'Afghanistan', 'Asia and developing Pacific', 'Southern Asia', 'Rest of Southern Asia', 'ldc'),\n",
-       " ('ALA', 'Åland Islands', 'Developed Countries', 'Europe', 'Northern and western Europe', 'developed'),\n",
-       " ('ALB', 'Albania', 'Developed Countries', 'Europe', 'Southern and eastern Europe', 'developed'),\n",
-       " ('DZA', 'Algeria', 'Africa', 'Africa', 'North Africa', 'developing'),\n",
-       " ('ASM', 'American Samoa', 'Asia and developing Pacific', 'South-East Asia and developing Pacific', 'Developing Pacific', 'developing')]"
-      ]
-     },
-     "execution_count": 15,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "engine.execute(\"select * from essd.regions\").fetchall()[0:5]"
+    "with engine.begin() as cxn:\n",
+    "    qres = cxn.execute(text(\"select * from essd.regions\")).fetchall()[0:5]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 11,
    "id": "c0728acf-ee26-4960-b34b-c098718774fd",
    "metadata": {},
    "outputs": [
@@ -510,7 +489,7 @@
        "[249 rows x 7 columns]"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -524,7 +503,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 12,
    "id": "005c82fd-ec54-4891-8c83-8310af84a783",
    "metadata": {},
    "outputs": [
@@ -833,7 +812,7 @@
        "[5123 rows x 6 columns]"
       ]
      },
-     "execution_count": 17,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -848,7 +827,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 13,
    "id": "8df08171-9cb3-4b66-af6a-403b93bdc9bb",
    "metadata": {},
    "outputs": [
@@ -953,7 +932,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.8"
+   "version": "3.11.5"
   }
  },
  "nbformat": 4,

diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,81 @@
+[project]
+name = "iso3166-ingest"
+version = "v1.0.0"
+description = "Example code and user interface for the iso3166 data pipeline."
+authors = [ { name = "Michael Tiemann", email = "72577720+MichaelTiemannOSC@users.noreply.github.com" } ]
+requires-python = ">=3.9"
+readme = "README.md"
+license = { text = "Apache-2.0" }
+keywords = [
+  "Climate",
+  "iso3166",
+  "Finance"
+]
+classifiers = [
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: MacOS",
+  "Operating System :: Microsoft :: Windows",
+  "Operating System :: Unix",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.9",
+  "Topic :: Office/Business :: Financial",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Software Development",
+]
+dependencies = [
+  "jupyterlab==4.0.4",
+  "osc-ingest-tools>=0.5.2",
+  "pycountry",
+  "pandas",
+  "numpy",
+]
+
+[project.urls]
+Homepage = "https://github.com/os-climate/iso3166-ingest"
+Repository = "https://github.com/os-climate/iso3166-ingest"
+Downloads = "https://github.com/os-climate/iso3166-ingest/releases"
+"Bug Tracker" = "https://github.com/os-climate/iso3166-ingest/issues"
+Documentation = "https://github.com/os-climate/iso3166-ingest/tree/main/docs"
+"Source Code" = "https://github.com/os-climate/iso3166-ingest"
+
+[build-system]
+requires = [ "pdm-backend" ]
+build-backend = "pdm.backend"
+
+[tool.pdm.scripts]
+pre_release = "scripts/dev-versioning.sh"
+release = "scripts/release-versioning.sh"
+test = "pytest"
+tox = "tox"
+doc = { shell = "cd docs && mkdocs serve", help = "Start the dev server for doc preview" }
+lint = "pre-commit run --all-files"
+complete = { call = "tasks.complete:main", help = "Create autocomplete files for bash and fish" }
+
+[tool.pdm.dev-dependencies]
+test = [
+  "pdm[pytest]",
+  "pdm[publish]",
+  "pytest-cov"
+]
+tox = [
+  "tox",
+  "tox-pdm>=0.5"
+]
+doc = [ "sphinx" ]
+dev = [
+  "tox>=4.11.3",
+  "tox-pdm>=0.7.0"
+]
+
+[tool.pytest.ini_options]
+testpaths = [ "test/" ]
+
+[tool.black]
+line-length = 120
diff --git a/test_environment.py b/test_environment.py
@@ -13,11 +13,7 @@ def main():
         raise ValueError("Unrecognized python interpreter: {}".format(REQUIRED_PYTHON))
 
     if system_major != required_major:
-        raise TypeError(
-            "This project requires Python {}. Found: Python {}".format(
-                required_major, sys.version
-            )
-        )
+        raise TypeError("This project requires Python {}. Found: Python {}".format(required_major, sys.version))
     else:
         print(">>> Development environment passes all tests!")