diff --git a/notebooks/iso3166-ingest.ipynb b/notebooks/iso3166-ingest.ipynb index 8d7ca0e..5ef8705 100644 --- a/notebooks/iso3166-ingest.ipynb +++ b/notebooks/iso3166-ingest.ipynb @@ -1,13 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "7dec07e8-1c7d-4864-8d7f-7b8079fc8aef", - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "id": "2e9358c2-843e-4c9f-8a72-01c1b46c82a7", @@ -57,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, "id": "3b5ed789-9a67-400f-b08d-bffb6e48e1af", "metadata": {}, "outputs": [], @@ -68,7 +60,8 @@ "import pathlib\n", "\n", "import trino\n", - "from sqlalchemy.engine import create_engine" + "from sqlalchemy.engine import create_engine\n", + "from sqlalchemy import text" ] }, { @@ -83,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "id": "36002a98-522f-43eb-91d8-75e92305f41c", "metadata": {}, "outputs": [], @@ -101,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 3, "id": "6fb8aedc-1d0b-4aff-9fce-901fde5c10d3", "metadata": {}, "outputs": [], @@ -121,7 +114,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, "id": "fb3678ba-ab2e-4d1e-b812-636315f0fc8c", "metadata": {}, "outputs": [], @@ -132,12 +125,12 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 5, "id": "3fa6bc63-4a78-4a4f-81a5-872ca3eb6577", "metadata": {}, "outputs": [], "source": [ - "trino_bucket = attach_s3_bucket(\"S3_DEV\")" + "trino_bucket = osc.attach_s3_bucket(\"S3_DEV\")" ] }, { @@ -150,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "id": "75e44b1a-b3fc-4de7-a8a3-0b7b5c806cd2", "metadata": {}, "outputs": [ @@ -171,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "id": "cf7ed364-7b63-4f85-9763-12da29ca7281", "metadata": {}, "outputs": [ @@ -225,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 8, "id": "f55ffe79-61ea-45f4-9b44-eccf0a74e879", "metadata": { "tags": [] @@ -254,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 9, "id": "40ccf11d-1694-46be-ab3b-61c991e259b5", "metadata": { "tags": [] @@ -265,8 +258,8 @@ "output_type": "stream", "text": [ "show tables in mdt_sandbox\n", - "[('assets_earnings_investments',), ('assets_usd_by_lei',), ('assets_xyz_by_lei',), ('average_fx',), ('cash_usd_by_lei',), ('cash_xyz_by_lei',), ('closing_fx',), ('customers_sales',), ('debt_equity_returns',), ('debt_usd_by_lei',), ('debt_xyz_by_lei',), ('emissions_targets',), ('employees',), ('expenditure_bills_burden',), ('financials_by_lei',), ('float_usd_by_lei',), ('float_xyz_by_lei',), ('fy_income_usd_by_lei',), ('fy_income_xyz_by_lei',), ('fy_revenue_usd_by_lei',), ('fy_revenue_xyz_by_lei',), ('housing_units_income',), ('itr_portfolio_universe',), ('net_plant_balance',), ('operations_emissions_by_fuel',), ('operations_emissions_by_tech',), ('portfolio_universe',), ('revenue_by_tech',), ('rmi_assets_earnings_investments',), ('rmi_assets_earnings_investments_source',), ('rmi_customers_sales',), ('rmi_customers_sales_source',), ('rmi_debt_equity_returns',), ('rmi_debt_equity_returns_source',), ('rmi_emissions_targets',), ('rmi_emissions_targets_source',), ('rmi_employees',), ('rmi_employees_source',), ('rmi_expenditure_bills_burden',), ('rmi_expenditure_bills_burden_source',), ('rmi_housing_units_income',), ('rmi_housing_units_income_source',), ('rmi_net_plant_balance',), ('rmi_net_plant_balance_source',), ('rmi_operations_emissions_by_fuel',), ('rmi_operations_emissions_by_fuel_source',), ('rmi_operations_emissions_by_tech',), ('rmi_operations_emissions_by_tech_source',), ('rmi_revenue_by_tech',), ('rmi_revenue_by_tech_source',), ('rmi_state_policies',), ('rmi_state_policies_source',), ('rmi_state_targets',), ('rmi_state_targets_source',), ('rmi_utility_information',), ('rmi_utility_information_2023',), ('rmi_utility_information_2023_source',), ('rmi_utility_information_source',), ('rmi_utility_state_map',), ('rmi_utility_state_map_2023',), ('rmi_utility_state_map_2023_source',), ('rmi_utility_state_map_source',), ('saved_queries',), ('sf_oecd_imgr_fco2',), ('sf_wdi_gdp',), ('state_policies',), ('state_targets',), ('utility_information',), ('utility_information_2023',), ('utility_state_map',), ('utility_state_map_2023',)]\n", - "[('assets_earnings_investments',), ('assets_usd_by_lei',), ('assets_xyz_by_lei',), ('average_fx',), ('cash_usd_by_lei',), ('cash_xyz_by_lei',), ('closing_fx',), ('customers_sales',), ('debt_equity_returns',), ('debt_usd_by_lei',), ('debt_xyz_by_lei',), ('emissions_targets',), ('employees',), ('expenditure_bills_burden',), ('financials_by_lei',), ('float_usd_by_lei',), ('float_xyz_by_lei',), ('fy_income_usd_by_lei',), ('fy_income_xyz_by_lei',), ('fy_revenue_usd_by_lei',), ('fy_revenue_xyz_by_lei',), ('housing_units_income',), ('itr_portfolio_universe',), ('net_plant_balance',), ('operations_emissions_by_fuel',), ('operations_emissions_by_tech',), ('portfolio_universe',), ('revenue_by_tech',), ('rmi_assets_earnings_investments',), ('rmi_assets_earnings_investments_source',), ('rmi_customers_sales',), ('rmi_customers_sales_source',), ('rmi_debt_equity_returns',), ('rmi_debt_equity_returns_source',), ('rmi_emissions_targets',), ('rmi_emissions_targets_source',), ('rmi_employees',), ('rmi_employees_source',), ('rmi_expenditure_bills_burden',), ('rmi_expenditure_bills_burden_source',), ('rmi_housing_units_income',), ('rmi_housing_units_income_source',), ('rmi_net_plant_balance',), ('rmi_net_plant_balance_source',), ('rmi_operations_emissions_by_fuel',), ('rmi_operations_emissions_by_fuel_source',), ('rmi_operations_emissions_by_tech',), ('rmi_operations_emissions_by_tech_source',), ('rmi_revenue_by_tech',), ('rmi_revenue_by_tech_source',), ('rmi_state_policies',), ('rmi_state_policies_source',), ('rmi_state_targets',), ('rmi_state_targets_source',), ('rmi_utility_information',), ('rmi_utility_information_2023',), ('rmi_utility_information_2023_source',), ('rmi_utility_information_source',), ('rmi_utility_state_map',), ('rmi_utility_state_map_2023',), ('rmi_utility_state_map_2023_source',), ('rmi_utility_state_map_source',), ('saved_queries',), ('sf_oecd_imgr_fco2',), ('sf_wdi_gdp',), ('state_policies',), ('state_targets',), ('utility_information',), ('utility_information_2023',), ('utility_state_map',), ('utility_state_map_2023',)]\n" + "[('isic_to_sector',), ('itr_portfolio_universe',), ('pcaf_dbt_models',), ('portfolio_universe',), ('saved_queries',), ('sf_oecd_exch_rates',), ('sf_oecd_exch_rates_source',), ('sf_oecd_exgr_dco2',), ('sf_oecd_exgr_dco2_source',), ('sf_oecd_imgr_fco2',), ('sf_oecd_imgr_fco2_source',), ('sf_primap_hist_emissions',), ('sf_primap_hist_emissions_source',), ('sf_total_sovereign_emissions',), ('sf_total_sovereign_emissions_source',), ('sf_unfccc_countries_source',), ('sf_unfccc_results',), ('sf_unfccc_results_source',), ('sf_unfccc_with_lulucf',), ('sf_unfccc_with_lulucf_source',), ('sf_unfccc_without_lulucf',), ('sf_unfccc_without_lulucf_source',), ('sf_wdi_gdp',), ('sf_wdi_gdp_source',), ('sf_wdi_population',), ('sf_wdi_population_source',)]\n", + "[('isic_to_sector',), ('itr_portfolio_universe',), ('pcaf_dbt_models',), ('portfolio_universe',), ('saved_queries',), ('sf_oecd_exch_rates',), ('sf_oecd_exch_rates_source',), ('sf_oecd_exgr_dco2',), ('sf_oecd_exgr_dco2_source',), ('sf_oecd_imgr_fco2',), ('sf_oecd_imgr_fco2_source',), ('sf_primap_hist_emissions',), ('sf_primap_hist_emissions_source',), ('sf_total_sovereign_emissions',), ('sf_total_sovereign_emissions_source',), ('sf_unfccc_countries_source',), ('sf_unfccc_results',), ('sf_unfccc_results_source',), ('sf_unfccc_with_lulucf',), ('sf_unfccc_with_lulucf_source',), ('sf_unfccc_without_lulucf',), ('sf_unfccc_without_lulucf_source',), ('sf_wdi_gdp',), ('sf_wdi_gdp_source',), ('sf_wdi_population',), ('sf_wdi_population_source',)]\n" ] } ], @@ -305,32 +298,18 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 10, "id": "711a5bf3-5299-4959-9c68-b5169507c2f4", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('AFG', 'Afghanistan', 'Asia and developing Pacific', 'Southern Asia', 'Rest of Southern Asia', 'ldc'),\n", - " ('ALA', 'Åland Islands', 'Developed Countries', 'Europe', 'Northern and western Europe', 'developed'),\n", - " ('ALB', 'Albania', 'Developed Countries', 'Europe', 'Southern and eastern Europe', 'developed'),\n", - " ('DZA', 'Algeria', 'Africa', 'Africa', 'North Africa', 'developing'),\n", - " ('ASM', 'American Samoa', 'Asia and developing Pacific', 'South-East Asia and developing Pacific', 'Developing Pacific', 'developing')]" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "engine.execute(\"select * from essd.regions\").fetchall()[0:5]" + "with engine.begin() as cxn:\n", + " qres = cxn.execute(text(\"select * from essd.regions\")).fetchall()[0:5]" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 11, "id": "c0728acf-ee26-4960-b34b-c098718774fd", "metadata": {}, "outputs": [ @@ -510,7 +489,7 @@ "[249 rows x 7 columns]" ] }, - "execution_count": 16, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -524,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 12, "id": "005c82fd-ec54-4891-8c83-8310af84a783", "metadata": {}, "outputs": [ @@ -833,7 +812,7 @@ "[5123 rows x 6 columns]" ] }, - "execution_count": 17, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -848,7 +827,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 13, "id": "8df08171-9cb3-4b66-af6a-403b93bdc9bb", "metadata": {}, "outputs": [ @@ -953,7 +932,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..28d5ca1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,81 @@ +[project] +name = "iso3166-ingest" +version = "v1.0.0" +description = "Example code and user interface for the iso3166 data pipeline." +authors = [ { name = "Michael Tiemann", email = "72577720+MichaelTiemannOSC@users.noreply.github.com" } ] +requires-python = ">=3.9" +readme = "README.md" +license = { text = "Apache-2.0" } +keywords = [ + "Climate", + "iso3166", + "Finance" +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.9", + "Topic :: Office/Business :: Financial", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", +] +dependencies = [ + "jupyterlab==4.0.4", + "osc-ingest-tools>=0.5.2", + "pycountry", + "pandas", + "numpy", +] + +[project.urls] +Homepage = "https://github.com/os-climate/iso3166-ingest" +Repository = "https://github.com/os-climate/iso3166-ingest" +Downloads = "https://github.com/os-climate/iso3166-ingest/releases" +"Bug Tracker" = "https://github.com/os-climate/iso3166-ingest/issues" +Documentation = "https://github.com/os-climate/iso3166-ingest/tree/main/docs" +"Source Code" = "https://github.com/os-climate/iso3166-ingest" + +[build-system] +requires = [ "pdm-backend" ] +build-backend = "pdm.backend" + +[tool.pdm.scripts] +pre_release = "scripts/dev-versioning.sh" +release = "scripts/release-versioning.sh" +test = "pytest" +tox = "tox" +doc = { shell = "cd docs && mkdocs serve", help = "Start the dev server for doc preview" } +lint = "pre-commit run --all-files" +complete = { call = "tasks.complete:main", help = "Create autocomplete files for bash and fish" } + +[tool.pdm.dev-dependencies] +test = [ + "pdm[pytest]", + "pdm[publish]", + "pytest-cov" +] +tox = [ + "tox", + "tox-pdm>=0.5" +] +doc = [ "sphinx" ] +dev = [ + "tox>=4.11.3", + "tox-pdm>=0.7.0" +] + +[tool.pytest.ini_options] +testpaths = [ "test/" ] + +[tool.black] +line-length = 120 diff --git a/test_environment.py b/test_environment.py index 5381850..7189443 100644 --- a/test_environment.py +++ b/test_environment.py @@ -13,11 +13,7 @@ def main(): raise ValueError("Unrecognized python interpreter: {}".format(REQUIRED_PYTHON)) if system_major != required_major: - raise TypeError( - "This project requires Python {}. Found: Python {}".format( - required_major, sys.version - ) - ) + raise TypeError("This project requires Python {}. Found: Python {}".format(required_major, sys.version)) else: print(">>> Development environment passes all tests!")