Skip to content

Commit

Permalink
chore: ⚡️ merge into world indicators
Browse files Browse the repository at this point in the history
  • Loading branch information
davidgasquez committed Mar 26, 2024
1 parent 268ff0e commit 5a4f178
Show file tree
Hide file tree
Showing 8 changed files with 254 additions and 78 deletions.
18 changes: 4 additions & 14 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"files.associations": {
"*.sql": "jinja-sql"
},
"[jinja-sql]": {
"editor.defaultFormatter": "innoverio.vscode-dbt-power-user",
"editor.formatOnSave": true
},
// "[jinja-sql]": {
// "editor.defaultFormatter": "innoverio.vscode-dbt-power-user",
// "editor.formatOnSave": true
// },
"[python]": {
"editor.defaultFormatter": "charliermarsh.ruff",
"editor.formatOnSave": true,
Expand All @@ -30,16 +30,6 @@
},
"python.analysis.typeCheckingMode": "basic",
"python.analysis.autoImportCompletions": true,
"sqltools.useNodeRuntime": true,
"sqltools.connections": [
{
"name": "DuckDB",
"accessMode": "Read Only",
"previewLimit": 50,
"driver": "DuckDB",
"databaseFilePath": "data/local.duckdb"
}
]
},
"extensions": [
"charliermarsh.ruff",
Expand Down
4 changes: 2 additions & 2 deletions datadex/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from dagster_dbt import DbtCliResource, load_assets_from_dbt_project
from dagster_duckdb_pandas import DuckDBPandasIOManager

from .assets import energy, huggingface
from .assets import others, indicators, huggingface
from .resources import IUCNRedListAPI, HuggingFaceResource

DBT_PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) + "/../dbt/"
Expand All @@ -13,7 +13,7 @@
dbt = DbtCliResource(project_dir=DBT_PROJECT_DIR, profiles_dir=DBT_PROJECT_DIR)

dbt_assets = load_assets_from_dbt_project(DBT_PROJECT_DIR, DBT_PROJECT_DIR)
all_assets = load_assets_from_modules([energy, huggingface])
all_assets = load_assets_from_modules([indicators, huggingface, others])

resources = {
"hf": HuggingFaceResource(token=EnvVar("HUGGINGFACE_TOKEN")),
Expand Down
7 changes: 2 additions & 5 deletions datadex/assets/huggingface.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import pandas as pd
from dagster import asset, AssetIn
from dagster import AssetIn, asset

from ..resources import HuggingFaceResource

Expand All @@ -16,13 +16,10 @@ def hf_asset(data: pd.DataFrame, hf: HuggingFaceResource) -> None:


datasets = [
"co2_global_trend",
"spain_energy_demand",
"owid_energy_data",
"owid_co2_data",
"wikidata_asteroids",
"threatened_animal_species",
"climate",
"country_year_indicators",
]

assets = []
Expand Down
24 changes: 24 additions & 0 deletions datadex/assets/indicators.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import pandas as pd
from dagster import asset


@asset
def owid_energy_data() -> pd.DataFrame:
"""
Raw Energy data from Our World in Data.
"""
energy_owid_url = (
"https://raw.githubusercontent.com/owid/energy-data/master/owid-energy-data.csv"
)
return pd.read_csv(energy_owid_url)


@asset
def owid_co2_data() -> pd.DataFrame:
"""
Raw CO2 data from Our World in Data.
"""
co2_owid_url = (
"https://raw.githubusercontent.com/owid/co2-data/master/owid-co2-data.csv"
)
return pd.read_csv(co2_owid_url)
31 changes: 0 additions & 31 deletions datadex/assets/energy.py → datadex/assets/others.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,37 +24,6 @@ def threatened_animal_species(iucn_redlist_api: IUCNRedListAPI) -> pd.DataFrame:
)


@asset
def owid_energy_data() -> pd.DataFrame:
"""
Raw Energy data from Our World in Data.
"""
energy_owid_url = (
"https://raw.githubusercontent.com/owid/energy-data/master/owid-energy-data.csv"
)
return pd.read_csv(energy_owid_url)


@asset
def owid_co2_data() -> pd.DataFrame:
"""
Raw CO2 data from Our World in Data.
"""
co2_owid_url = (
"https://raw.githubusercontent.com/owid/co2-data/master/owid-co2-data.csv"
)
return pd.read_csv(co2_owid_url)


@asset
def co2_global_trend() -> pd.DataFrame:
"""
Trends in Atmospheric Carbon Dioxide from NOAA/ESRL.
"""
co2_noaa_url = "https://gml.noaa.gov/webdata/ccgg/trends/co2/co2_trend_gl.csv"
return pd.read_csv(co2_noaa_url, skiprows=24)


@asset
def wikidata_asteroids() -> pd.DataFrame:
"""
Expand Down
18 changes: 0 additions & 18 deletions dbt/models/climate.sql

This file was deleted.

222 changes: 222 additions & 0 deletions dbt/models/country_year_indicators.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
with
owid_energy_data as (
select
country,
year,
iso_code,
population,
gdp,
biofuel_cons_change_pct,
biofuel_cons_change_twh,
biofuel_cons_per_capita,
biofuel_consumption,
biofuel_elec_per_capita,
biofuel_electricity,
biofuel_share_elec,
biofuel_share_energy,
carbon_intensity_elec,
coal_cons_change_pct,
coal_cons_change_twh,
coal_cons_per_capita,
coal_consumption,
coal_elec_per_capita,
coal_electricity,
coal_prod_change_pct,
coal_prod_change_twh,
coal_prod_per_capita,
coal_production,
coal_share_elec,
coal_share_energy,
electricity_demand,
electricity_generation,
electricity_share_energy,
energy_cons_change_pct,
energy_cons_change_twh,
energy_per_capita,
energy_per_gdp,
fossil_cons_change_pct,
fossil_cons_change_twh,
fossil_elec_per_capita,
fossil_electricity,
fossil_energy_per_capita,
fossil_fuel_consumption,
fossil_share_elec,
fossil_share_energy,
gas_cons_change_pct,
gas_cons_change_twh,
gas_consumption,
gas_elec_per_capita,
gas_electricity,
gas_energy_per_capita,
gas_prod_change_pct,
gas_prod_change_twh,
gas_prod_per_capita,
gas_production,
gas_share_elec,
gas_share_energy,
greenhouse_gas_emissions,
hydro_cons_change_pct,
hydro_cons_change_twh,
hydro_consumption,
hydro_elec_per_capita,
hydro_electricity,
hydro_energy_per_capita,
hydro_share_elec,
hydro_share_energy,
low_carbon_cons_change_pct,
low_carbon_cons_change_twh,
low_carbon_consumption,
low_carbon_elec_per_capita,
low_carbon_electricity,
low_carbon_energy_per_capita,
low_carbon_share_elec,
low_carbon_share_energy,
net_elec_imports,
net_elec_imports_share_demand,
nuclear_cons_change_pct,
nuclear_cons_change_twh,
nuclear_consumption,
nuclear_elec_per_capita,
nuclear_electricity,
nuclear_energy_per_capita,
nuclear_share_elec,
nuclear_share_energy,
oil_cons_change_pct,
oil_cons_change_twh,
oil_consumption,
oil_elec_per_capita,
oil_electricity,
oil_energy_per_capita,
oil_prod_change_pct,
oil_prod_change_twh,
oil_prod_per_capita,
oil_production,
oil_share_elec,
oil_share_energy,
other_renewable_consumption,
other_renewable_electricity,
other_renewable_exc_biofuel_electricity,
other_renewables_cons_change_pct,
other_renewables_cons_change_twh,
other_renewables_elec_per_capita,
other_renewables_elec_per_capita_exc_biofuel,
other_renewables_energy_per_capita,
other_renewables_share_elec,
other_renewables_share_elec_exc_biofuel,
other_renewables_share_energy,
per_capita_electricity,
primary_energy_consumption,
renewables_cons_change_pct,
renewables_cons_change_twh,
renewables_consumption,
renewables_elec_per_capita,
renewables_electricity,
renewables_energy_per_capita,
renewables_share_elec,
renewables_share_energy,
solar_cons_change_pct,
solar_cons_change_twh,
solar_consumption,
solar_elec_per_capita,
solar_electricity,
solar_energy_per_capita,
solar_share_elec,
solar_share_energy,
wind_cons_change_pct,
wind_cons_change_twh,
wind_consumption,
wind_elec_per_capita,
wind_electricity,
wind_energy_per_capita,
wind_share_elec,
wind_share_energy
from {{ source("main", "owid_energy_data") }}
),

owid_co2_data as (
select
country,
year,
iso_code,
population,
gdp,
cement_co2,
cement_co2_per_capita,
co2,
co2_growth_abs,
co2_growth_prct,
co2_including_luc,
co2_including_luc_growth_abs,
co2_including_luc_growth_prct,
co2_including_luc_per_capita,
co2_including_luc_per_gdp,
co2_including_luc_per_unit_energy,
co2_per_capita,
co2_per_gdp,
co2_per_unit_energy,
coal_co2,
coal_co2_per_capita,
consumption_co2,
consumption_co2_per_capita,
consumption_co2_per_gdp,
cumulative_cement_co2,
cumulative_co2,
cumulative_co2_including_luc,
cumulative_coal_co2,
cumulative_flaring_co2,
cumulative_gas_co2,
cumulative_luc_co2,
cumulative_oil_co2,
cumulative_other_co2,
energy_per_capita,
energy_per_gdp,
flaring_co2,
flaring_co2_per_capita,
gas_co2,
gas_co2_per_capita,
ghg_excluding_lucf_per_capita,
ghg_per_capita,
land_use_change_co2,
land_use_change_co2_per_capita,
methane,
methane_per_capita,
nitrous_oxide,
nitrous_oxide_per_capita,
oil_co2,
oil_co2_per_capita,
other_co2_per_capita,
other_industry_co2,
primary_energy_consumption,
share_global_cement_co2,
share_global_co2,
share_global_co2_including_luc,
share_global_coal_co2,
share_global_cumulative_cement_co2,
share_global_cumulative_co2,
share_global_cumulative_co2_including_luc,
share_global_cumulative_coal_co2,
share_global_cumulative_flaring_co2,
share_global_cumulative_gas_co2,
share_global_cumulative_luc_co2,
share_global_cumulative_oil_co2,
share_global_cumulative_other_co2,
share_global_flaring_co2,
share_global_gas_co2,
share_global_luc_co2,
share_global_oil_co2,
share_global_other_co2,
share_of_temperature_change_from_ghg,
temperature_change_from_ch4,
temperature_change_from_co2,
temperature_change_from_ghg,
temperature_change_from_n2o,
total_ghg,
total_ghg_excluding_lucf,
trade_co2,
trade_co2_share
from {{ source("main", "owid_co2_data") }}
)

select e.*, c.*
from owid_energy_data as e
join owid_co2_data as c on e.iso_code = c.iso_code and e.year = c.year
8 changes: 0 additions & 8 deletions dbt/models/sources.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@ version: 2
sources:
- name: main
tables:
- name: threatened_animal_species
meta:
dagster:
asset_key: ["threatened_animal_species"]
- name: owid_co2_data
meta:
dagster:
Expand All @@ -15,7 +11,3 @@ sources:
meta:
dagster:
asset_key: ["owid_energy_data"]
- name: co2_global_trend
meta:
dagster:
asset_key: ["co2_global_trend"]

0 comments on commit 5a4f178

Please sign in to comment.