From f8e015c528826062d055e78f32d22e3f98ae007a Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:43:49 -0700 Subject: [PATCH 01/10] scripts to process CTPS landuse and skims --- notebooks/process_landuse.ipynb | 1402 +++++++++++++++++++++++++++++++ notebooks/process_skims.ipynb | 587 +++++++++++++ 2 files changed, 1989 insertions(+) create mode 100644 notebooks/process_landuse.ipynb create mode 100644 notebooks/process_skims.ipynb diff --git a/notebooks/process_landuse.ipynb b/notebooks/process_landuse.ipynb new file mode 100644 index 0000000..d28e582 --- /dev/null +++ b/notebooks/process_landuse.ipynb @@ -0,0 +1,1402 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import geopandas as gpd\n", + "import sys\n", + "import os\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", + "output_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\\processed\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## inputs" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "block_to_taz_file = os.path.join(input_dir, \"zonal\", \"shp\", \"taz_2010block_allocation_20230314.csv\")\n", + "\n", + "# zone shape file\n", + "zone_shape_file = os.path.join(input_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\")\n", + "\n", + "# emp input files\n", + "ma_emp_input_file = os.path.join(input_dir, \"zonal\", \"ma_employment_run_113-209_2019_v20250829.csv\")\n", + "nhri_emp_input_file = os.path.join(input_dir, \"zonal\", \"nhri_employment_2020_v20230518.csv\")\n", + "\n", + "# pop input files\n", + "ma_pop_input_file = os.path.join(input_dir, \"zonal\", \"ma_population_run_113-209_2019_v20250829.csv\")\n", + "nhri_pop_input_file = os.path.join(input_dir, \"zonal\", \"nhri_population_2020_v20230518.csv\")\n", + "\n", + "# enrollment input files\n", + "enroll_file = os.path.join(input_dir, \"zonal\", \"enroll_v20240709.csv\")\n", + "\n", + "# parking\n", + "parking_file = os.path.join(input_dir, \"zonal\", \"parking_v20221007.csv\")\n", + "\n", + "# terminal times\n", + "terminal_time_file = os.path.join(input_dir, \"zonal\", \"terminal_times_v20221118.csv\")\n", + "\n", + "# county FIPS\n", + "ma_county_fips_file = os.path.join(input_dir, \"tl_2025_25_cousub\", \"tl_2025_25_cousub.shp\")\n", + "nh_county_fips_file = os.path.join(input_dir, \"tl_2020_33_cousub\", \"tl_2020_33_cousub.shp\")\n", + "ri_county_fips_file = os.path.join(input_dir, \"tl_2022_44_cousub\", \"tl_2022_44_cousub.shp\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "block_to_taz_df = pd.read_csv(block_to_taz_file)\n", + "\n", + "zone_shape_df = gpd.read_file(zone_shape_file)\n", + "\n", + "ma_emp_df = pd.read_csv(ma_emp_input_file)\n", + "nhri_emp_df = pd.read_csv(nhri_emp_input_file)\n", + "\n", + "ma_pop_df = pd.read_csv(ma_pop_input_file)\n", + "nhri_pop_df = pd.read_csv(nhri_pop_input_file)\n", + "\n", + "enroll_df = pd.read_csv(enroll_file)\n", + "parking_df = pd.read_csv(parking_file)\n", + "terminal_time_df = pd.read_csv(terminal_time_file)\n", + "\n", + "ma_county_fips_df = gpd.read_file(ma_county_fips_file)\n", + "nh_county_fips_df = gpd.read_file(nh_county_fips_file)\n", + "ri_county_fips_df = gpd.read_file(ri_county_fips_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "prepare land use attributs \n", + "https://github.com/CTPSSTAFF/lighthouse/blob/e8479d64a4e50506fc9fdd42890e89e8446a59be/model/configs/settings.yaml#L58-L90" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## zonal data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Assumptions\n", + "- COUNTY: county FIPS, externals use 99\n", + "- DISTRICT: district from shp\n", + "- SD: state from shp\n", + "- TOTACRE: total_area from shp, convert to acre\n", + "- RESACRE: TOTACRE*0.5\n", + "- CIACRE: TOTACRE*0.1\n", + "- area_type: urban from shp\n", + "- TOPOLOGY: 1 for all\n", + "- TERMINAL: 4 for urban, 1 or others" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "county_gdfs = []\n", + "\n", + "for state_gdf, state_name in [(ma_county_fips_df, \"MA\"), (nh_county_fips_df, \"NH\"), (ri_county_fips_df, \"RI\")]:\n", + " if state_gdf is not None:\n", + " county_gdfs.append(state_gdf)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "combined_counties = pd.concat(county_gdfs, ignore_index=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "EPSG:4269\n", + "EPSG:26986\n" + ] + } + ], + "source": [ + "print(combined_counties.crs)\n", + "print(zone_shape_df.crs)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "combined_counties = combined_counties.to_crs(zone_shape_df.crs)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "taz_centroids = zone_shape_df.copy()\n", + "taz_centroids[\"geometry\"] = taz_centroids.geometry.centroid\n", + "\n", + "result = gpd.sjoin(\n", + " taz_centroids, \n", + " combined_counties[[ \"geometry\", \"COUNTYFP\" ]], \n", + " how=\"left\", \n", + " predicate=\"within\"\n", + ")\n", + "result_grouped = result.groupby(result.index).first()\n", + "\n", + "# county id\n", + "zone_shape_df[\"COUNTY\"] = result_grouped[\"COUNTYFP\"].astype('Int64')\n", + "zone_shape_df.loc[zone_shape_df[\"type\"] == 'E', \"COUNTY\"] = 99\n", + "\n", + "# square miles to acres\n", + "# TODO: update RESACRE and CIACRE with actual land use data\n", + "zone_shape_df[\"TOTACRE\"] = zone_shape_df[\"total_area\"] * 640\n", + "zone_shape_df[\"RESACRE\"] = zone_shape_df[\"TOTACRE\"] * 0.5\n", + "zone_shape_df[\"CIACRE\"] = zone_shape_df[\"TOTACRE\"] * 0.1\n", + "\n", + "# superdistricts\n", + "zone_shape_df[\"SD\"] = 99\n", + "zone_shape_df.loc[zone_shape_df['state']==\"MA\", 'SD'] = 25\n", + "zone_shape_df.loc[zone_shape_df['state']==\"NH\", 'SD'] = 33\n", + "zone_shape_df.loc[zone_shape_df['state']==\"RI\", 'SD'] = 44\n", + "\n", + "# TODO: update area type\n", + "zone_shape_df.rename(columns={\"urban\": \"area_type\"}, inplace=True)\n", + "\n", + "# TODO: update terminal times\n", + "zone_shape_df[\"TERMINAL\"] = np.where(zone_shape_df[\"area_type\"] == 1, 4, 1)\n", + "\n", + "zone_shape_df.rename(columns={\"district\": \"DISTRICT\"}, inplace=True)\n", + "zone_shape_df.rename(columns={\"taz_id\": \"TAZ\"}, inplace=True)\n", + "zone_shape_df[\"TOPOLOGY\"] = 1" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df[[\"TAZ\", \"COUNTY\", \"DISTRICT\", \"SD\", \"TOTACRE\", \"RESACRE\", \"CIACRE\", \"area_type\", \"TERMINAL\", \"TOPOLOGY\"]]\n", + "zone_shape_df = zone_shape_df.sort_values(\"TAZ\").reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "TAZ", + "rawType": "int32", + "type": "integer" + }, + { + "name": "COUNTY", + "rawType": "Int64", + "type": "integer" + }, + { + "name": "DISTRICT", + "rawType": "int32", + "type": "integer" + }, + { + "name": "SD", + "rawType": "int64", + "type": "integer" + }, + { + "name": "TOTACRE", + "rawType": "float64", + "type": "float" + }, + { + "name": "RESACRE", + "rawType": "float64", + "type": "float" + }, + { + "name": "CIACRE", + "rawType": "float64", + "type": "float" + }, + { + "name": "area_type", + "rawType": "int32", + "type": "integer" + }, + { + "name": "TERMINAL", + "rawType": "int64", + "type": "integer" + }, + { + "name": "TOPOLOGY", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "9abdbdd2-8c10-4d19-a514-74d21ca550c1", + "rows": [ + [ + "0", + "1", + "25", + "0", + "25", + "45.44", + "22.72", + "4.544", + "1", + "4", + "1" + ], + [ + "1", + "2", + "25", + "0", + "25", + "14.079999999999998", + "7.039999999999999", + "1.408", + "1", + "4", + "1" + ], + [ + "2", + "3", + "25", + "0", + "25", + "19.84", + "9.92", + "1.984", + "1", + "4", + "1" + ], + [ + "3", + "4", + "25", + "0", + "25", + "17.92", + "8.96", + "1.7920000000000003", + "1", + "4", + "1" + ], + [ + "4", + "5", + "25", + "0", + "25", + "27.519999999999996", + "13.759999999999998", + "2.752", + "1", + "4", + "1" + ], + [ + "5", + "6", + "25", + "0", + "25", + "18.560000000000002", + "9.280000000000001", + "1.8560000000000003", + "1", + "4", + "1" + ], + [ + "6", + "7", + "25", + "0", + "25", + "23.68", + "11.84", + "2.368", + "1", + "4", + "1" + ], + [ + "7", + "8", + "25", + "0", + "25", + "12.16", + "6.08", + "1.2160000000000002", + "1", + "4", + "1" + ], + [ + "8", + "9", + "25", + "0", + "25", + "10.24", + "5.12", + "1.024", + "1", + "4", + "1" + ], + [ + "9", + "10", + "25", + "0", + "25", + "15.36", + "7.68", + "1.536", + "1", + "4", + "1" + ], + [ + "10", + "11", + "25", + "0", + "25", + "17.28", + "8.64", + "1.7280000000000002", + "1", + "4", + "1" + ], + [ + "11", + "12", + "25", + "0", + "25", + "21.76", + "10.88", + "2.176", + "1", + "4", + "1" + ], + [ + "12", + "13", + "25", + "0", + "25", + "20.48", + "10.24", + "2.048", + "1", + "4", + "1" + ], + [ + "13", + "14", + "25", + "0", + "25", + "9.6", + "4.8", + "0.96", + "1", + "4", + "1" + ], + [ + "14", + "15", + "25", + "0", + "25", + "8.96", + "4.48", + "0.8960000000000001", + "1", + "4", + "1" + ], + [ + "15", + "16", + "25", + "0", + "25", + "2.56", + "1.28", + "0.256", + "1", + "4", + "1" + ], + [ + "16", + "17", + "25", + "0", + "25", + "8.96", + "4.48", + "0.8960000000000001", + "1", + "4", + "1" + ], + [ + "17", + "18", + "25", + "0", + "25", + "6.4", + "3.2", + "0.6400000000000001", + "1", + "4", + "1" + ], + [ + "18", + "19", + "25", + "0", + "25", + "28.799999999999997", + "14.399999999999999", + "2.88", + "1", + "4", + "1" + ], + [ + "19", + "20", + "25", + "0", + "25", + "44.160000000000004", + "22.080000000000002", + "4.416", + "1", + "4", + "1" + ], + [ + "20", + "21", + "25", + "0", + "25", + "51.2", + "25.6", + "5.120000000000001", + "1", + "4", + "1" + ], + [ + "21", + "22", + "25", + "0", + "25", + "53.120000000000005", + "26.560000000000002", + "5.312000000000001", + "1", + "4", + "1" + ], + [ + "22", + "23", + "25", + "0", + "25", + "9.6", + "4.8", + "0.96", + "1", + "4", + "1" + ], + [ + "23", + "24", + "25", + "0", + "25", + "11.52", + "5.76", + "1.152", + "1", + "4", + "1" + ], + [ + "24", + "25", + "25", + "0", + "25", + "43.52", + "21.76", + "4.352", + "1", + "4", + "1" + ], + [ + "25", + "26", + "25", + "0", + "25", + "62.72", + "31.36", + "6.272", + "1", + "4", + "1" + ], + [ + "26", + "27", + "25", + "0", + "25", + "15.36", + "7.68", + "1.536", + "1", + "4", + "1" + ], + [ + "27", + "28", + "25", + "0", + "25", + "28.159999999999997", + "14.079999999999998", + "2.816", + "1", + "4", + "1" + ], + [ + "28", + "29", + "25", + "0", + "25", + "12.16", + "6.08", + "1.2160000000000002", + "1", + "4", + "1" + ], + [ + "29", + "30", + "25", + "0", + "25", + "9.6", + "4.8", + "0.96", + "1", + "4", + "1" + ], + [ + "30", + "31", + "25", + "0", + "25", + "9.6", + "4.8", + "0.96", + "1", + "4", + "1" + ], + [ + "31", + "32", + "25", + "0", + "25", + "14.719999999999999", + "7.359999999999999", + "1.472", + "1", + "4", + "1" + ], + [ + "32", + "33", + "25", + "0", + "25", + "6.4", + "3.2", + "0.6400000000000001", + "1", + "4", + "1" + ], + [ + "33", + "34", + "25", + "0", + "25", + "3.2", + "1.6", + "0.32000000000000006", + "1", + "4", + "1" + ], + [ + "34", + "35", + "25", + "0", + "25", + "8.32", + "4.16", + "0.8320000000000001", + "1", + "4", + "1" + ], + [ + "35", + "36", + "25", + "0", + "25", + "21.76", + "10.88", + "2.176", + "1", + "4", + "1" + ], + [ + "36", + "37", + "25", + "0", + "25", + "10.88", + "5.44", + "1.088", + "1", + "4", + "1" + ], + [ + "37", + "38", + "25", + "0", + "25", + "3.84", + "1.92", + "0.384", + "1", + "4", + "1" + ], + [ + "38", + "39", + "25", + "0", + "25", + "14.719999999999999", + "7.359999999999999", + "1.472", + "1", + "4", + "1" + ], + [ + "39", + "40", + "25", + "0", + "25", + "5.76", + "2.88", + "0.576", + "1", + "4", + "1" + ], + [ + "40", + "41", + "25", + "0", + "25", + "7.039999999999999", + "3.5199999999999996", + "0.704", + "1", + "4", + "1" + ], + [ + "41", + "42", + "25", + "0", + "25", + "1.92", + "0.96", + "0.192", + "1", + "4", + "1" + ], + [ + "42", + "43", + "25", + "0", + "25", + "1.92", + "0.96", + "0.192", + "1", + "4", + "1" + ], + [ + "43", + "44", + "25", + "0", + "25", + "2.56", + "1.28", + "0.256", + "1", + "4", + "1" + ], + [ + "44", + "45", + "25", + "0", + "25", + "5.12", + "2.56", + "0.512", + "1", + "4", + "1" + ], + [ + "45", + "46", + "25", + "0", + "25", + "5.76", + "2.88", + "0.576", + "1", + "4", + "1" + ], + [ + "46", + "47", + "25", + "0", + "25", + "5.76", + "2.88", + "0.576", + "1", + "4", + "1" + ], + [ + "47", + "48", + "25", + "0", + "25", + "4.48", + "2.24", + "0.44800000000000006", + "1", + "4", + "1" + ], + [ + "48", + "49", + "25", + "0", + "25", + "1.92", + "0.96", + "0.192", + "1", + "4", + "1" + ], + [ + "49", + "50", + "25", + "0", + "25", + "12.8", + "6.4", + "1.2800000000000002", + "1", + "4", + "1" + ] + ], + "shape": { + "columns": 10, + "rows": 5839 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TAZCOUNTYDISTRICTSDTOTACRERESACRECIACREarea_typeTERMINALTOPOLOGY
012502545.4422.724.544141
122502514.087.041.408141
232502519.849.921.984141
342502517.928.961.792141
452502527.5213.762.752141
.................................
58342090969989990.000.000.000011
58352090979989990.000.000.000011
58362090989989990.000.000.000011
58372090999989990.000.000.000011
58382091009989990.000.000.000011
\n", + "

5839 rows × 10 columns

\n", + "
" + ], + "text/plain": [ + " TAZ COUNTY DISTRICT SD TOTACRE RESACRE CIACRE area_type \\\n", + "0 1 25 0 25 45.44 22.72 4.544 1 \n", + "1 2 25 0 25 14.08 7.04 1.408 1 \n", + "2 3 25 0 25 19.84 9.92 1.984 1 \n", + "3 4 25 0 25 17.92 8.96 1.792 1 \n", + "4 5 25 0 25 27.52 13.76 2.752 1 \n", + "... ... ... ... .. ... ... ... ... \n", + "5834 209096 99 89 99 0.00 0.00 0.000 0 \n", + "5835 209097 99 89 99 0.00 0.00 0.000 0 \n", + "5836 209098 99 89 99 0.00 0.00 0.000 0 \n", + "5837 209099 99 89 99 0.00 0.00 0.000 0 \n", + "5838 209100 99 89 99 0.00 0.00 0.000 0 \n", + "\n", + " TERMINAL TOPOLOGY \n", + "0 4 1 \n", + "1 4 1 \n", + "2 4 1 \n", + "3 4 1 \n", + "4 4 1 \n", + "... ... ... \n", + "5834 1 1 \n", + "5835 1 1 \n", + "5836 1 1 \n", + "5837 1 1 \n", + "5838 1 1 \n", + "\n", + "[5839 rows x 10 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "zone_shape_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## employment" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def process_employment_data(employment_df, block_to_taz_df):\n", + " merged = employment_df.merge(block_to_taz_df, on='block_id', how='left')\n", + "\n", + " employment_columns = [col for col in merged.columns \n", + " if col not in ['block_id', 'taz_id', 'area_fct']]\n", + "\n", + " # apply area factor\n", + " for col in employment_columns:\n", + " merged[col] = merged[col] * merged['area_fct']\n", + "\n", + " taz_employment = merged.groupby('taz_id')[employment_columns].sum().reset_index()\n", + "\n", + " taz_employment['RETEMPN'] = taz_employment['6_ret_leis']\n", + " taz_employment['FPSEMPN'] = taz_employment['3_finance'] + taz_employment['9_profbus']\n", + " taz_employment['HEREMPN'] = taz_employment['2_eduhlth']\n", + " taz_employment['OTHEMPN'] = taz_employment['1_constr'] + taz_employment['4_public'] + taz_employment['5_info'] + taz_employment['8_other']\n", + " taz_employment['AGREMPN'] = 0 \n", + " taz_employment['MWTEMPN'] = taz_employment['7_manu'] + taz_employment['10_ttu']\n", + "\n", + " taz_employment = taz_employment[['taz_id', 'total_households', 'total_jobs', 'RETEMPN', 'FPSEMPN', 'HEREMPN', 'OTHEMPN', 'AGREMPN', 'MWTEMPN']]\n", + " taz_employment = taz_employment.rename(\n", + " columns={'total_households': 'TOTHH', \n", + " 'total_jobs': 'TOTEMP',\n", + " 'taz_id': 'TAZ'}\n", + " )\n", + "\n", + " taz_employment = taz_employment.sort_values('TAZ').reset_index(drop=True)\n", + "\n", + " return taz_employment" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "ma_emp_agg = process_employment_data(ma_emp_df, block_to_taz_df)\n", + "nhri_emp_agg = process_employment_data(nhri_emp_df, block_to_taz_df)\n", + "\n", + "emp_combined = pd.concat([ma_emp_agg, nhri_emp_agg], ignore_index=False)\n", + "emp_combined = emp_combined.set_index('TAZ')\n", + "emp_combined = emp_combined.groupby(level=0).sum().reset_index()\n", + "emp_combined = emp_combined.sort_values('TAZ').reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(emp_combined, on='TAZ', how='left').fillna(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## population" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def process_population_data(population_df, block_to_taz_df):\n", + " merged = population_df.merge(block_to_taz_df, on='block_id', how='left')\n", + "\n", + " merged['is_age_0519'] = ((merged['age'] >= 5) & (merged['age'] <= 19)).astype(int)\n", + "\n", + " # apply area factor\n", + " merged['pop_weighted'] = merged['area_fct']\n", + " merged['pop_age0519_weighted'] = merged['area_fct'] * merged['is_age_0519']\n", + " \n", + " taz_population = merged.groupby('taz_id').agg(\n", + " TOTPOP=('pop_weighted', 'sum'),\n", + " AGE0519=('pop_age0519_weighted', 'sum')\n", + " ).reset_index()\n", + " \n", + " taz_population = taz_population.rename(\n", + " columns={'taz_id': 'TAZ'}\n", + " )\n", + " taz_population = taz_population.sort_values('TAZ').reset_index(drop=True)\n", + "\n", + " return taz_population" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "ma_pop_agg = process_population_data(ma_pop_df, block_to_taz_df)\n", + "nhri_pop_agg = process_population_data(nhri_pop_df, block_to_taz_df)\n", + "\n", + "pop_combined = pd.concat([ma_pop_agg, nhri_pop_agg], ignore_index=False)\n", + "pop_combined = pop_combined.set_index('TAZ')\n", + "pop_combined = pop_combined.groupby(level=0).sum().reset_index()\n", + "pop_combined = pop_combined.sort_values('TAZ').reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(pop_combined, on='TAZ', how='left').fillna(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## parking" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(\n", + " parking_df[['taz_id','cost_hr']].rename(columns={'taz_id': 'TAZ'}),\n", + " on='TAZ', \n", + " how='left').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: update parking cost\n", + "zone_shape_df.rename(columns={'cost_hr': 'PRKCST'}, inplace=True)\n", + "zone_shape_df['OPRKCST'] = zone_shape_df['PRKCST']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## enrollment" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "enroll_df['HSENROLL'] = enroll_df['k12']\n", + "\n", + "# TODO: update with actual enrollment data\n", + "enroll_df['COLLFTE'] = (enroll_df['college_total'] * 0.9).astype(int)\n", + "enroll_df['COLLPTE'] = (enroll_df['college_total'] * 0.1).astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(\n", + " enroll_df[['taz_id','HSENROLL', 'COLLFTE', 'COLLPTE']].rename(columns={'taz_id': 'TAZ'}),\n", + " on='TAZ', \n", + " how='left').fillna(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## output" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df[[\"TAZ\", \"COUNTY\", \"DISTRICT\", \"SD\", \n", + " \"TOTHH\", \"TOTPOP\", \"TOTACRE\", \"RESACRE\", \"CIACRE\", \"TOTEMP\", \"AGE0519\",\n", + " \"RETEMPN\", \"FPSEMPN\", \"HEREMPN\", \"OTHEMPN\", \"AGREMPN\", \"MWTEMPN\",\n", + " \"PRKCST\", \"OPRKCST\", \n", + " \"area_type\", \"HSENROLL\", \"COLLFTE\", \"COLLPTE\",\n", + " \"TERMINAL\", \"TOPOLOGY\"\n", + " ]].to_csv(os.path.join(output_dir, \"land_use.csv\"), index=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "myenv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/process_skims.ipynb b/notebooks/process_skims.ipynb new file mode 100644 index 0000000..f039a17 --- /dev/null +++ b/notebooks/process_skims.ipynb @@ -0,0 +1,587 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import openmatrix as omx\n", + "import numpy as np\n", + "import os\n", + "\n", + "import openmatrix\n", + "openmatrix.numpy = np" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "input_file_name = \"ta_md.omx\"\n", + "output_file_name = \"ta_md_ev.omx\"\n", + "\n", + "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\_skim\"\n", + "output_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\"\n", + "\n", + "input_file = os.path.join(input_dir, input_file_name)\n", + "output_file = os.path.join(output_dir, output_file_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['ID', 'RCIndex']" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "matrix_in = omx.open_file(input_file, 'r')\n", + "matrix_in.list_mappings()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "shape = matrix_in[matrix_in.list_matrices()[0]].shape\n", + "shape = (int(shape[0]), int(shape[1]))\n", + "\n", + "matrix_out = omx.open_file(output_file, 'w', shape=shape)\n", + "\n", + "for mapping_name in matrix_in.list_mappings():\n", + " if mapping_name == \"RCIndex\" or mapping_name == \"Destination\":\n", + " old_mapping = matrix_in.mapping(mapping_name)\n", + " old_index = [\n", + " key for key, value in sorted(old_mapping.items(), key=lambda item: item[1])\n", + " ]\n", + " matrix_out.create_mapping(\"ID\", old_index)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process higway_am" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process da_time\n", + "Saved as da_time__AM\n", + "Transposed and saved as da_time__PM\n", + "process sr_time\n", + "Saved as sr_time__AM\n", + "Transposed and saved as sr_time__PM\n", + "process da_toll\n", + "Saved as da_toll__AM\n", + "Transposed and saved as da_toll__PM\n", + "process sr_toll\n", + "Saved as sr_toll__AM\n", + "Transposed and saved as sr_toll__PM\n", + "process dist\n", + "Saved as dist__AM\n", + "Transposed and saved as dist__PM\n", + "process rs_fare\n", + "Saved as rs_fare__AM\n", + "Transposed and saved as rs_fare__PM\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'da_time', \n", + " 'sr_time', \n", + " 'da_toll',\n", + " 'sr_toll',\n", + " 'dist',\n", + " 'rs_fare',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " am_name = f\"{matrix_name}__AM\"\n", + " matrix_out[am_name] = data\n", + " print(f\"Saved as {am_name}\")\n", + " \n", + " pm_name = f\"{matrix_name}__PM\"\n", + " # transpose the matrix\n", + " matrix_out[pm_name] = data.T\n", + " print(f\"Transposed and saved as {pm_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process highway_md" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process da_time\n", + "Saved as da_time__MD\n", + "Copied and saved as da_time__EV\n", + "process sr_time\n", + "Saved as sr_time__MD\n", + "Copied and saved as sr_time__EV\n", + "process da_toll\n", + "Saved as da_toll__MD\n", + "Copied and saved as da_toll__EV\n", + "process sr_toll\n", + "Saved as sr_toll__MD\n", + "Copied and saved as sr_toll__EV\n", + "process dist\n", + "Saved as dist__MD\n", + "Copied and saved as dist__EV\n", + "process rs_fare\n", + "Saved as rs_fare__MD\n", + "Copied and saved as rs_fare__EV\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'da_time', \n", + " 'sr_time', \n", + " 'da_toll',\n", + " 'sr_toll',\n", + " 'dist',\n", + " 'rs_fare',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " md_name = f\"{matrix_name}__MD\"\n", + " matrix_out[md_name] = data\n", + " print(f\"Saved as {md_name}\")\n", + " \n", + " ev_name = f\"{matrix_name}__EV\"\n", + " # copy the matrix\n", + " matrix_out[ev_name] = data.copy()\n", + " print(f\"Copied and saved as {ev_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process nm_daily" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process dist\n", + "Saved as dist_nm\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'dist'\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " new_name = \"dist_nm\"\n", + " matrix_out[new_name] = data\n", + " print(f\"Saved as {new_name}\")\n", + " \n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process tw_am" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process xfer\n", + "Saved as tw_xfer__AM\n", + "Transposed and saved as tw_xfer__PM\n", + "process Fare\n", + "Saved as tw_fare__AM\n", + "Transposed and saved as tw_fare__PM\n", + "process iwait\n", + "Saved as tw_iwait__AM\n", + "Transposed and saved as tw_iwait__PM\n", + "process xwait\n", + "Saved as tw_xwait__AM\n", + "Transposed and saved as tw_xwait__PM\n", + "process ivtt\n", + "Saved as tw_ivtt__AM\n", + "Transposed and saved as tw_ivtt__PM\n", + "process walk\n", + "Saved as tw_walk__AM\n", + "Transposed and saved as tw_walk__PM\n", + "process gen_cost\n", + "Saved as tw_gen_cost__AM\n", + "Transposed and saved as tw_gen_cost__PM\n", + "process tdist\n", + "Saved as tw_tdist__AM\n", + "Transposed and saved as tw_tdist__PM\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'xfer',\n", + " 'Fare',\n", + " 'iwait',\n", + " 'xwait',\n", + " 'ivtt',\n", + " 'walk',\n", + " 'gen_cost',\n", + " 'tdist',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " matrix_name = matrix_name.lower()\n", + "\n", + " am_name = f\"tw_{matrix_name}__AM\"\n", + " matrix_out[am_name] = data\n", + " print(f\"Saved as {am_name}\")\n", + " \n", + " pm_name = f\"tw_{matrix_name}__PM\"\n", + " # Transpose the matrix\n", + " matrix_out[pm_name] = data.T\n", + " print(f\"Transposed and saved as {pm_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process tw_md" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process xfer\n", + "Saved as tw_xfer__MD\n", + "Copied and saved as tw_xfer__EV\n", + "process Fare\n", + "Saved as tw_fare__MD\n", + "Copied and saved as tw_fare__EV\n", + "process iwait\n", + "Saved as tw_iwait__MD\n", + "Copied and saved as tw_iwait__EV\n", + "process xwait\n", + "Saved as tw_xwait__MD\n", + "Copied and saved as tw_xwait__EV\n", + "process ivtt\n", + "Saved as tw_ivtt__MD\n", + "Copied and saved as tw_ivtt__EV\n", + "process walk\n", + "Saved as tw_walk__MD\n", + "Copied and saved as tw_walk__EV\n", + "process gen_cost\n", + "Saved as tw_gen_cost__MD\n", + "Copied and saved as tw_gen_cost__EV\n", + "process tdist\n", + "Saved as tw_tdist__MD\n", + "Copied and saved as tw_tdist__EV\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'xfer',\n", + " 'Fare',\n", + " 'iwait',\n", + " 'xwait',\n", + " 'ivtt',\n", + " 'walk',\n", + " 'gen_cost',\n", + " 'tdist',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " matrix_name = matrix_name.lower()\n", + "\n", + " md_name = f\"tw_{matrix_name}__MD\"\n", + " matrix_out[md_name] = data\n", + " print(f\"Saved as {md_name}\")\n", + " \n", + " ev_name = f\"tw_{matrix_name}__EV\"\n", + " matrix_out[ev_name] = data.copy()\n", + " print(f\"Copied and saved as {ev_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process ta_am" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process xfer\n", + "Saved as ta_xfer__AM\n", + "Transposed and saved as ta_xfer__PM\n", + "process Fare\n", + "Saved as ta_fare__AM\n", + "Transposed and saved as ta_fare__PM\n", + "process iwait\n", + "Saved as ta_iwait__AM\n", + "Transposed and saved as ta_iwait__PM\n", + "process xwait\n", + "Saved as ta_xwait__AM\n", + "Transposed and saved as ta_xwait__PM\n", + "process ivtt\n", + "Saved as ta_ivtt__AM\n", + "Transposed and saved as ta_ivtt__PM\n", + "process walk\n", + "Saved as ta_walk__AM\n", + "Transposed and saved as ta_walk__PM\n", + "process dtime\n", + "Saved as ta_dtime__AM\n", + "Transposed and saved as ta_dtime__PM\n", + "process ddist\n", + "Saved as ta_ddist__AM\n", + "Transposed and saved as ta_ddist__PM\n", + "process tdist\n", + "Saved as ta_tdist__AM\n", + "Transposed and saved as ta_tdist__PM\n", + "process gen_cost\n", + "Saved as ta_gen_cost__AM\n", + "Transposed and saved as ta_gen_cost__PM\n", + "process auto_cost\n", + "Saved as ta_auto_cost__AM\n", + "Transposed and saved as ta_auto_cost__PM\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'xfer',\n", + " 'Fare',\n", + " 'iwait',\n", + " 'xwait',\n", + " 'ivtt',\n", + " 'walk',\n", + " 'dtime',\n", + " 'ddist',\n", + " 'tdist',\n", + " 'gen_cost',\n", + " 'auto_cost',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " matrix_name = matrix_name.lower()\n", + "\n", + " am_name = f\"ta_{matrix_name}__AM\"\n", + " matrix_out[am_name] = data\n", + " print(f\"Saved as {am_name}\")\n", + " \n", + " pm_name = f\"ta_{matrix_name}__PM\"\n", + " # Transpose the matrix\n", + " matrix_out[pm_name] = data.T\n", + " print(f\"Transposed and saved as {pm_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## process ta_md" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process xfer\n", + "Saved as ta_xfer__MD\n", + "Copied and saved as ta_xfer__EV\n", + "process Fare\n", + "Saved as ta_fare__MD\n", + "Copied and saved as ta_fare__EV\n", + "process iwait\n", + "Saved as ta_iwait__MD\n", + "Copied and saved as ta_iwait__EV\n", + "process xwait\n", + "Saved as ta_xwait__MD\n", + "Copied and saved as ta_xwait__EV\n", + "process ivtt\n", + "Saved as ta_ivtt__MD\n", + "Copied and saved as ta_ivtt__EV\n", + "process walk\n", + "Saved as ta_walk__MD\n", + "Copied and saved as ta_walk__EV\n", + "process dtime\n", + "Saved as ta_dtime__MD\n", + "Copied and saved as ta_dtime__EV\n", + "process ddist\n", + "Saved as ta_ddist__MD\n", + "Copied and saved as ta_ddist__EV\n", + "process tdist\n", + "Saved as ta_tdist__MD\n", + "Copied and saved as ta_tdist__EV\n", + "process gen_cost\n", + "Saved as ta_gen_cost__MD\n", + "Copied and saved as ta_gen_cost__EV\n", + "process auto_cost\n", + "Saved as ta_auto_cost__MD\n", + "Copied and saved as ta_auto_cost__EV\n" + ] + } + ], + "source": [ + "matrices_to_process = [\n", + " 'xfer',\n", + " 'Fare',\n", + " 'iwait',\n", + " 'xwait',\n", + " 'ivtt',\n", + " 'walk',\n", + " 'dtime',\n", + " 'ddist',\n", + " 'tdist',\n", + " 'gen_cost',\n", + " 'auto_cost',\n", + "]\n", + "\n", + "for matrix_name in matrices_to_process:\n", + "\n", + " print(f\"process {matrix_name}\")\n", + " \n", + " data = matrix_in[matrix_name][:]\n", + " matrix_name = matrix_name.lower()\n", + "\n", + " md_name = f\"ta_{matrix_name}__MD\"\n", + " matrix_out[md_name] = data\n", + " print(f\"Saved as {md_name}\")\n", + " \n", + " ev_name = f\"ta_{matrix_name}__EV\"\n", + " matrix_out[ev_name] = data.copy()\n", + " print(f\"Copied and saved as {ev_name}\")\n", + "\n", + "matrix_in.close()\n", + "matrix_out.close()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "myenv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 17dedc30547b30f42d69c8486e4f911f3073e7f2 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Mon, 8 Jun 2026 16:15:44 -0700 Subject: [PATCH 02/10] use transit accessibility to represent area_type --- notebooks/process_landuse.ipynb | 236 ++++++++++++++++++-------------- 1 file changed, 133 insertions(+), 103 deletions(-) diff --git a/notebooks/process_landuse.ipynb b/notebooks/process_landuse.ipynb index d28e582..8cecc2c 100644 --- a/notebooks/process_landuse.ipynb +++ b/notebooks/process_landuse.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -57,16 +57,18 @@ "\n", "# terminal times\n", "terminal_time_file = os.path.join(input_dir, \"zonal\", \"terminal_times_v20221118.csv\")\n", + "# transit access density\n", + "access_density_file = os.path.join(input_dir, \"zonal\", \"access_density.csv\")\n", "\n", "# county FIPS\n", "ma_county_fips_file = os.path.join(input_dir, \"tl_2025_25_cousub\", \"tl_2025_25_cousub.shp\")\n", "nh_county_fips_file = os.path.join(input_dir, \"tl_2020_33_cousub\", \"tl_2020_33_cousub.shp\")\n", - "ri_county_fips_file = os.path.join(input_dir, \"tl_2022_44_cousub\", \"tl_2022_44_cousub.shp\")\n" + "ri_county_fips_file = os.path.join(input_dir, \"tl_2022_44_cousub\", \"tl_2022_44_cousub.shp\")" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -83,10 +85,12 @@ "enroll_df = pd.read_csv(enroll_file)\n", "parking_df = pd.read_csv(parking_file)\n", "terminal_time_df = pd.read_csv(terminal_time_file)\n", + "access_density_df = pd.read_csv(access_density_file)\n", "\n", "ma_county_fips_df = gpd.read_file(ma_county_fips_file)\n", "nh_county_fips_df = gpd.read_file(nh_county_fips_file)\n", - "ri_county_fips_df = gpd.read_file(ri_county_fips_file)" + "ri_county_fips_df = gpd.read_file(ri_county_fips_file)\n", + "\n" ] }, { @@ -122,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -135,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -144,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -163,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -172,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -203,12 +207,6 @@ "zone_shape_df.loc[zone_shape_df['state']==\"NH\", 'SD'] = 33\n", "zone_shape_df.loc[zone_shape_df['state']==\"RI\", 'SD'] = 44\n", "\n", - "# TODO: update area type\n", - "zone_shape_df.rename(columns={\"urban\": \"area_type\"}, inplace=True)\n", - "\n", - "# TODO: update terminal times\n", - "zone_shape_df[\"TERMINAL\"] = np.where(zone_shape_df[\"area_type\"] == 1, 4, 1)\n", - "\n", "zone_shape_df.rename(columns={\"district\": \"DISTRICT\"}, inplace=True)\n", "zone_shape_df.rename(columns={\"taz_id\": \"TAZ\"}, inplace=True)\n", "zone_shape_df[\"TOPOLOGY\"] = 1" @@ -216,7 +214,39 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "# area type and terminal times\n", + "access_density_df = access_density_df.merge(\n", + " terminal_time_df[[\"access_density\",\"terminal_time_p\"]],\n", + " on=\"access_density\",\n", + " how=\"left\"\n", + ")\n", + "\n", + "access_density_df.rename(\n", + " columns={\"taz_id\": \"TAZ\", \"access_density\": \"area_type\", \"terminal_time_p\": \"TERMINAL\"}, \n", + " inplace=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(\n", + " access_density_df[[\"TAZ\", \"area_type\", \"TERMINAL\"]],\n", + " on=\"TAZ\",\n", + " how=\"left\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -226,7 +256,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -275,7 +305,7 @@ }, { "name": "area_type", - "rawType": "int32", + "rawType": "int64", "type": "integer" }, { @@ -289,7 +319,7 @@ "type": "integer" } ], - "ref": "9abdbdd2-8c10-4d19-a514-74d21ca550c1", + "ref": "9b342455-ca72-417d-b207-71bbedf25be8", "rows": [ [ "0", @@ -301,7 +331,7 @@ "22.72", "4.544", "1", - "4", + "5", "1" ], [ @@ -314,7 +344,7 @@ "7.039999999999999", "1.408", "1", - "4", + "5", "1" ], [ @@ -327,7 +357,7 @@ "9.92", "1.984", "1", - "4", + "5", "1" ], [ @@ -340,7 +370,7 @@ "8.96", "1.7920000000000003", "1", - "4", + "5", "1" ], [ @@ -352,7 +382,7 @@ "27.519999999999996", "13.759999999999998", "2.752", - "1", + "2", "4", "1" ], @@ -366,7 +396,7 @@ "9.280000000000001", "1.8560000000000003", "1", - "4", + "5", "1" ], [ @@ -378,7 +408,7 @@ "23.68", "11.84", "2.368", - "1", + "2", "4", "1" ], @@ -392,7 +422,7 @@ "6.08", "1.2160000000000002", "1", - "4", + "5", "1" ], [ @@ -405,7 +435,7 @@ "5.12", "1.024", "1", - "4", + "5", "1" ], [ @@ -418,7 +448,7 @@ "7.68", "1.536", "1", - "4", + "5", "1" ], [ @@ -431,7 +461,7 @@ "8.64", "1.7280000000000002", "1", - "4", + "5", "1" ], [ @@ -444,7 +474,7 @@ "10.88", "2.176", "1", - "4", + "5", "1" ], [ @@ -457,7 +487,7 @@ "10.24", "2.048", "1", - "4", + "5", "1" ], [ @@ -470,7 +500,7 @@ "4.8", "0.96", "1", - "4", + "5", "1" ], [ @@ -483,7 +513,7 @@ "4.48", "0.8960000000000001", "1", - "4", + "5", "1" ], [ @@ -496,7 +526,7 @@ "1.28", "0.256", "1", - "4", + "5", "1" ], [ @@ -509,7 +539,7 @@ "4.48", "0.8960000000000001", "1", - "4", + "5", "1" ], [ @@ -522,7 +552,7 @@ "3.2", "0.6400000000000001", "1", - "4", + "5", "1" ], [ @@ -535,7 +565,7 @@ "14.399999999999999", "2.88", "1", - "4", + "5", "1" ], [ @@ -548,7 +578,7 @@ "22.080000000000002", "4.416", "1", - "4", + "5", "1" ], [ @@ -561,7 +591,7 @@ "25.6", "5.120000000000001", "1", - "4", + "5", "1" ], [ @@ -574,7 +604,7 @@ "26.560000000000002", "5.312000000000001", "1", - "4", + "5", "1" ], [ @@ -587,7 +617,7 @@ "4.8", "0.96", "1", - "4", + "5", "1" ], [ @@ -600,7 +630,7 @@ "5.76", "1.152", "1", - "4", + "5", "1" ], [ @@ -612,7 +642,7 @@ "43.52", "21.76", "4.352", - "1", + "2", "4", "1" ], @@ -625,7 +655,7 @@ "62.72", "31.36", "6.272", - "1", + "2", "4", "1" ], @@ -639,7 +669,7 @@ "7.68", "1.536", "1", - "4", + "5", "1" ], [ @@ -652,7 +682,7 @@ "14.079999999999998", "2.816", "1", - "4", + "5", "1" ], [ @@ -665,7 +695,7 @@ "6.08", "1.2160000000000002", "1", - "4", + "5", "1" ], [ @@ -678,7 +708,7 @@ "4.8", "0.96", "1", - "4", + "5", "1" ], [ @@ -691,7 +721,7 @@ "4.8", "0.96", "1", - "4", + "5", "1" ], [ @@ -704,7 +734,7 @@ "7.359999999999999", "1.472", "1", - "4", + "5", "1" ], [ @@ -717,7 +747,7 @@ "3.2", "0.6400000000000001", "1", - "4", + "5", "1" ], [ @@ -730,7 +760,7 @@ "1.6", "0.32000000000000006", "1", - "4", + "5", "1" ], [ @@ -743,7 +773,7 @@ "4.16", "0.8320000000000001", "1", - "4", + "5", "1" ], [ @@ -756,7 +786,7 @@ "10.88", "2.176", "1", - "4", + "5", "1" ], [ @@ -769,7 +799,7 @@ "5.44", "1.088", "1", - "4", + "5", "1" ], [ @@ -782,7 +812,7 @@ "1.92", "0.384", "1", - "4", + "5", "1" ], [ @@ -795,7 +825,7 @@ "7.359999999999999", "1.472", "1", - "4", + "5", "1" ], [ @@ -808,7 +838,7 @@ "2.88", "0.576", "1", - "4", + "5", "1" ], [ @@ -821,7 +851,7 @@ "3.5199999999999996", "0.704", "1", - "4", + "5", "1" ], [ @@ -834,7 +864,7 @@ "0.96", "0.192", "1", - "4", + "5", "1" ], [ @@ -847,7 +877,7 @@ "0.96", "0.192", "1", - "4", + "5", "1" ], [ @@ -860,7 +890,7 @@ "1.28", "0.256", "1", - "4", + "5", "1" ], [ @@ -873,7 +903,7 @@ "2.56", "0.512", "1", - "4", + "5", "1" ], [ @@ -886,7 +916,7 @@ "2.88", "0.576", "1", - "4", + "5", "1" ], [ @@ -899,7 +929,7 @@ "2.88", "0.576", "1", - "4", + "5", "1" ], [ @@ -912,7 +942,7 @@ "2.24", "0.44800000000000006", "1", - "4", + "5", "1" ], [ @@ -925,7 +955,7 @@ "0.96", "0.192", "1", - "4", + "5", "1" ], [ @@ -938,7 +968,7 @@ "6.4", "1.2800000000000002", "1", - "4", + "5", "1" ] ], @@ -989,7 +1019,7 @@ " 22.72\n", " 4.544\n", " 1\n", - " 4\n", + " 5\n", " 1\n", " \n", " \n", @@ -1002,7 +1032,7 @@ " 7.04\n", " 1.408\n", " 1\n", - " 4\n", + " 5\n", " 1\n", " \n", " \n", @@ -1015,7 +1045,7 @@ " 9.92\n", " 1.984\n", " 1\n", - " 4\n", + " 5\n", " 1\n", " \n", " \n", @@ -1028,7 +1058,7 @@ " 8.96\n", " 1.792\n", " 1\n", - " 4\n", + " 5\n", " 1\n", " \n", " \n", @@ -1040,7 +1070,7 @@ " 27.52\n", " 13.76\n", " 2.752\n", - " 1\n", + " 2\n", " 4\n", " 1\n", " \n", @@ -1066,7 +1096,7 @@ " 0.00\n", " 0.00\n", " 0.000\n", - " 0\n", + " 6\n", " 1\n", " 1\n", " \n", @@ -1079,7 +1109,7 @@ " 0.00\n", " 0.00\n", " 0.000\n", - " 0\n", + " 6\n", " 1\n", " 1\n", " \n", @@ -1092,7 +1122,7 @@ " 0.00\n", " 0.00\n", " 0.000\n", - " 0\n", + " 6\n", " 1\n", " 1\n", " \n", @@ -1105,7 +1135,7 @@ " 0.00\n", " 0.00\n", " 0.000\n", - " 0\n", + " 6\n", " 1\n", " 1\n", " \n", @@ -1118,7 +1148,7 @@ " 0.00\n", " 0.00\n", " 0.000\n", - " 0\n", + " 6\n", " 1\n", " 1\n", " \n", @@ -1133,19 +1163,19 @@ "1 2 25 0 25 14.08 7.04 1.408 1 \n", "2 3 25 0 25 19.84 9.92 1.984 1 \n", "3 4 25 0 25 17.92 8.96 1.792 1 \n", - "4 5 25 0 25 27.52 13.76 2.752 1 \n", + "4 5 25 0 25 27.52 13.76 2.752 2 \n", "... ... ... ... .. ... ... ... ... \n", - "5834 209096 99 89 99 0.00 0.00 0.000 0 \n", - "5835 209097 99 89 99 0.00 0.00 0.000 0 \n", - "5836 209098 99 89 99 0.00 0.00 0.000 0 \n", - "5837 209099 99 89 99 0.00 0.00 0.000 0 \n", - "5838 209100 99 89 99 0.00 0.00 0.000 0 \n", + "5834 209096 99 89 99 0.00 0.00 0.000 6 \n", + "5835 209097 99 89 99 0.00 0.00 0.000 6 \n", + "5836 209098 99 89 99 0.00 0.00 0.000 6 \n", + "5837 209099 99 89 99 0.00 0.00 0.000 6 \n", + "5838 209100 99 89 99 0.00 0.00 0.000 6 \n", "\n", " TERMINAL TOPOLOGY \n", - "0 4 1 \n", - "1 4 1 \n", - "2 4 1 \n", - "3 4 1 \n", + "0 5 1 \n", + "1 5 1 \n", + "2 5 1 \n", + "3 5 1 \n", "4 4 1 \n", "... ... ... \n", "5834 1 1 \n", @@ -1157,7 +1187,7 @@ "[5839 rows x 10 columns]" ] }, - "execution_count": 11, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1175,7 +1205,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -1212,7 +1242,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -1227,7 +1257,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -1243,7 +1273,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -1271,7 +1301,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -1286,7 +1316,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -1302,7 +1332,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -1314,7 +1344,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -1332,7 +1362,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -1345,7 +1375,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -1364,7 +1394,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ From d4cd1181b3fefc750e65acf9b55985439fa371dd Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:26:02 -0700 Subject: [PATCH 03/10] prepare subarea inputs --- notebooks/prepare_subrea_inputs.ipynb | 277 ++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 notebooks/prepare_subrea_inputs.ipynb diff --git a/notebooks/prepare_subrea_inputs.ipynb b/notebooks/prepare_subrea_inputs.ipynb new file mode 100644 index 0000000..ea6eb71 --- /dev/null +++ b/notebooks/prepare_subrea_inputs.ipynb @@ -0,0 +1,277 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import geopandas as gpd\n", + "import sys\n", + "import os\n", + "import numpy as np\n", + "\n", + "import openmatrix as omx\n", + "\n", + "import openmatrix\n", + "openmatrix.numpy = np" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "landuse_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", + "skim_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## subarea land use " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "zone_file = gpd.read_file(os.path.join(landuse_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\"))\n", + "landuse_file = pd.read_csv(os.path.join(landuse_dir, \"processed\", \"land_use.csv\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "downtown_taz_ids = zone_file[zone_file[\"ring\"]==0].taz_id.to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "landuse_file = landuse_file[landuse_file[\"TAZ\"].isin(downtown_taz_ids)]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "landuse_file.to_csv(os.path.join(landuse_dir, \"processed\", \"land_use_subarea.csv\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## subarea skims" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "skims = [\n", + " \"hwy_am_pm\",\n", + " \"hwy_md_ev\",\n", + " \"nm_daily\",\n", + " \"tw_am_pm\",\n", + " \"tw_md_ev\",\n", + " \"ta_am_pm\",\n", + " \"ta_md_ev\",\n", + "]\n", + "\n", + "mapping_name = \"ID\"" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_am_pm.omx\n", + "save matrix da_time__AM\n", + "save matrix da_time__PM\n", + "save matrix da_toll__AM\n", + "save matrix da_toll__PM\n", + "save matrix dist__AM\n", + "save matrix dist__PM\n", + "save matrix rs_fare__AM\n", + "save matrix rs_fare__PM\n", + "save matrix sr_time__AM\n", + "save matrix sr_time__PM\n", + "save matrix sr_toll__AM\n", + "save matrix sr_toll__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_md_ev.omx\n", + "save matrix da_time__EV\n", + "save matrix da_time__MD\n", + "save matrix da_toll__EV\n", + "save matrix da_toll__MD\n", + "save matrix dist__EV\n", + "save matrix dist__MD\n", + "save matrix rs_fare__EV\n", + "save matrix rs_fare__MD\n", + "save matrix sr_time__EV\n", + "save matrix sr_time__MD\n", + "save matrix sr_toll__EV\n", + "save matrix sr_toll__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\nm_daily.omx\n", + "save matrix dist_nm\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\nm_daily_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_am_pm.omx\n", + "save matrix tw_fare__AM\n", + "save matrix tw_fare__PM\n", + "save matrix tw_gen_cost__AM\n", + "save matrix tw_gen_cost__PM\n", + "save matrix tw_ivtt__AM\n", + "save matrix tw_ivtt__PM\n", + "save matrix tw_iwait__AM\n", + "save matrix tw_iwait__PM\n", + "save matrix tw_tdist__AM\n", + "save matrix tw_tdist__PM\n", + "save matrix tw_walk__AM\n", + "save matrix tw_walk__PM\n", + "save matrix tw_xfer__AM\n", + "save matrix tw_xfer__PM\n", + "save matrix tw_xwait__AM\n", + "save matrix tw_xwait__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_md_ev.omx\n", + "save matrix tw_fare__EV\n", + "save matrix tw_fare__MD\n", + "save matrix tw_gen_cost__EV\n", + "save matrix tw_gen_cost__MD\n", + "save matrix tw_ivtt__EV\n", + "save matrix tw_ivtt__MD\n", + "save matrix tw_iwait__EV\n", + "save matrix tw_iwait__MD\n", + "save matrix tw_tdist__EV\n", + "save matrix tw_tdist__MD\n", + "save matrix tw_walk__EV\n", + "save matrix tw_walk__MD\n", + "save matrix tw_xfer__EV\n", + "save matrix tw_xfer__MD\n", + "save matrix tw_xwait__EV\n", + "save matrix tw_xwait__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_am_pm.omx\n", + "save matrix ta_auto_cost__AM\n", + "save matrix ta_auto_cost__PM\n", + "save matrix ta_ddist__AM\n", + "save matrix ta_ddist__PM\n", + "save matrix ta_dtime__AM\n", + "save matrix ta_dtime__PM\n", + "save matrix ta_fare__AM\n", + "save matrix ta_fare__PM\n", + "save matrix ta_gen_cost__AM\n", + "save matrix ta_gen_cost__PM\n", + "save matrix ta_ivtt__AM\n", + "save matrix ta_ivtt__PM\n", + "save matrix ta_iwait__AM\n", + "save matrix ta_iwait__PM\n", + "save matrix ta_tdist__AM\n", + "save matrix ta_tdist__PM\n", + "save matrix ta_walk__AM\n", + "save matrix ta_walk__PM\n", + "save matrix ta_xfer__AM\n", + "save matrix ta_xfer__PM\n", + "save matrix ta_xwait__AM\n", + "save matrix ta_xwait__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_md_ev.omx\n", + "save matrix ta_auto_cost__EV\n", + "save matrix ta_auto_cost__MD\n", + "save matrix ta_ddist__EV\n", + "save matrix ta_ddist__MD\n", + "save matrix ta_dtime__EV\n", + "save matrix ta_dtime__MD\n", + "save matrix ta_fare__EV\n", + "save matrix ta_fare__MD\n", + "save matrix ta_gen_cost__EV\n", + "save matrix ta_gen_cost__MD\n", + "save matrix ta_ivtt__EV\n", + "save matrix ta_ivtt__MD\n", + "save matrix ta_iwait__EV\n", + "save matrix ta_iwait__MD\n", + "save matrix ta_tdist__EV\n", + "save matrix ta_tdist__MD\n", + "save matrix ta_walk__EV\n", + "save matrix ta_walk__MD\n", + "save matrix ta_xfer__EV\n", + "save matrix ta_xfer__MD\n", + "save matrix ta_xwait__EV\n", + "save matrix ta_xwait__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_md_ev_subarea.omx\n" + ] + } + ], + "source": [ + "for skim in skims:\n", + " input_omx = os.path.join(skim_dir, \"processed\", f\"{skim}.omx\")\n", + " output_omx = os.path.join(skim_dir, \"processed\", \"subarea\", f\"{skim}_subarea.omx\")\n", + "\n", + " print(f\"process {input_omx}\")\n", + "\n", + " with omx.open_file(input_omx, \"r\") as matrix_in:\n", + " mapping = matrix_in.mapping(mapping_name)\n", + " zones = [z for z in downtown_taz_ids if z in mapping]\n", + " locations = [mapping[z] for z in zones]\n", + " \n", + " zones = np.array(zones, dtype=np.int32)\n", + " locations = np.array(locations)\n", + "\n", + " with omx.open_file(output_omx, \"w\") as matrix_out:\n", + " matrix_out.create_mapping(mapping_name, zones)\n", + "\n", + " for matrix_name in matrix_in.list_matrices():\n", + " data = matrix_in[matrix_name][:]\n", + " subarea_data = data[np.ix_(locations,locations)]\n", + " matrix_out[matrix_name] = subarea_data\n", + "\n", + " print(f\"save matrix {matrix_name}\")\n", + " print(f\"create {output_omx}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "myenv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 60d687d60355989b6d626cd280e27e0751663889 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:44:39 -0700 Subject: [PATCH 04/10] use ring=0 or 1 to select subarea --- notebooks/prepare_subrea_inputs.ipynb | 216 ++++++++++---------------- 1 file changed, 82 insertions(+), 134 deletions(-) diff --git a/notebooks/prepare_subrea_inputs.ipynb b/notebooks/prepare_subrea_inputs.ipynb index ea6eb71..eed73cc 100644 --- a/notebooks/prepare_subrea_inputs.ipynb +++ b/notebooks/prepare_subrea_inputs.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -20,9 +20,19 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ "landuse_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", "skim_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\"" @@ -37,9 +47,19 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ "zone_file = gpd.read_file(os.path.join(landuse_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\"))\n", "landuse_file = pd.read_csv(os.path.join(landuse_dir, \"processed\", \"land_use.csv\"))" @@ -47,29 +67,59 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ - "downtown_taz_ids = zone_file[zone_file[\"ring\"]==0].taz_id.to_list()" + "downtown_taz_ids = zone_file[zone_file[\"ring\"].isin([0,1])].taz_id.to_list()" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ "landuse_file = landuse_file[landuse_file[\"TAZ\"].isin(downtown_taz_ids)]" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ - "landuse_file.to_csv(os.path.join(landuse_dir, \"processed\", \"land_use_subarea.csv\"))" + "landuse_file.to_csv(os.path.join(landuse_dir, \"processed\", \"land_use_subarea.csv\"), index=False)" ] }, { @@ -81,9 +131,19 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], "source": [ "skims = [\n", " \"hwy_am_pm\",\n", @@ -100,128 +160,16 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_am_pm.omx\n", - "save matrix da_time__AM\n", - "save matrix da_time__PM\n", - "save matrix da_toll__AM\n", - "save matrix da_toll__PM\n", - "save matrix dist__AM\n", - "save matrix dist__PM\n", - "save matrix rs_fare__AM\n", - "save matrix rs_fare__PM\n", - "save matrix sr_time__AM\n", - "save matrix sr_time__PM\n", - "save matrix sr_toll__AM\n", - "save matrix sr_toll__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_md_ev.omx\n", - "save matrix da_time__EV\n", - "save matrix da_time__MD\n", - "save matrix da_toll__EV\n", - "save matrix da_toll__MD\n", - "save matrix dist__EV\n", - "save matrix dist__MD\n", - "save matrix rs_fare__EV\n", - "save matrix rs_fare__MD\n", - "save matrix sr_time__EV\n", - "save matrix sr_time__MD\n", - "save matrix sr_toll__EV\n", - "save matrix sr_toll__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_md_ev_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\nm_daily.omx\n", - "save matrix dist_nm\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\nm_daily_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_am_pm.omx\n", - "save matrix tw_fare__AM\n", - "save matrix tw_fare__PM\n", - "save matrix tw_gen_cost__AM\n", - "save matrix tw_gen_cost__PM\n", - "save matrix tw_ivtt__AM\n", - "save matrix tw_ivtt__PM\n", - "save matrix tw_iwait__AM\n", - "save matrix tw_iwait__PM\n", - "save matrix tw_tdist__AM\n", - "save matrix tw_tdist__PM\n", - "save matrix tw_walk__AM\n", - "save matrix tw_walk__PM\n", - "save matrix tw_xfer__AM\n", - "save matrix tw_xfer__PM\n", - "save matrix tw_xwait__AM\n", - "save matrix tw_xwait__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_md_ev.omx\n", - "save matrix tw_fare__EV\n", - "save matrix tw_fare__MD\n", - "save matrix tw_gen_cost__EV\n", - "save matrix tw_gen_cost__MD\n", - "save matrix tw_ivtt__EV\n", - "save matrix tw_ivtt__MD\n", - "save matrix tw_iwait__EV\n", - "save matrix tw_iwait__MD\n", - "save matrix tw_tdist__EV\n", - "save matrix tw_tdist__MD\n", - "save matrix tw_walk__EV\n", - "save matrix tw_walk__MD\n", - "save matrix tw_xfer__EV\n", - "save matrix tw_xfer__MD\n", - "save matrix tw_xwait__EV\n", - "save matrix tw_xwait__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_md_ev_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_am_pm.omx\n", - "save matrix ta_auto_cost__AM\n", - "save matrix ta_auto_cost__PM\n", - "save matrix ta_ddist__AM\n", - "save matrix ta_ddist__PM\n", - "save matrix ta_dtime__AM\n", - "save matrix ta_dtime__PM\n", - "save matrix ta_fare__AM\n", - "save matrix ta_fare__PM\n", - "save matrix ta_gen_cost__AM\n", - "save matrix ta_gen_cost__PM\n", - "save matrix ta_ivtt__AM\n", - "save matrix ta_ivtt__PM\n", - "save matrix ta_iwait__AM\n", - "save matrix ta_iwait__PM\n", - "save matrix ta_tdist__AM\n", - "save matrix ta_tdist__PM\n", - "save matrix ta_walk__AM\n", - "save matrix ta_walk__PM\n", - "save matrix ta_xfer__AM\n", - "save matrix ta_xfer__PM\n", - "save matrix ta_xwait__AM\n", - "save matrix ta_xwait__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_md_ev.omx\n", - "save matrix ta_auto_cost__EV\n", - "save matrix ta_auto_cost__MD\n", - "save matrix ta_ddist__EV\n", - "save matrix ta_ddist__MD\n", - "save matrix ta_dtime__EV\n", - "save matrix ta_dtime__MD\n", - "save matrix ta_fare__EV\n", - "save matrix ta_fare__MD\n", - "save matrix ta_gen_cost__EV\n", - "save matrix ta_gen_cost__MD\n", - "save matrix ta_ivtt__EV\n", - "save matrix ta_ivtt__MD\n", - "save matrix ta_iwait__EV\n", - "save matrix ta_iwait__MD\n", - "save matrix ta_tdist__EV\n", - "save matrix ta_tdist__MD\n", - "save matrix ta_walk__EV\n", - "save matrix ta_walk__MD\n", - "save matrix ta_xfer__EV\n", - "save matrix ta_xfer__MD\n", - "save matrix ta_xwait__EV\n", - "save matrix ta_xwait__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_md_ev_subarea.omx\n" + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mnotebook controller is DISPOSED. \n", + "\u001b[1;31mView Jupyter log for further details." ] } ], From 8a98711d98c2e8a92f350b28c73e26ee044dfe41 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:11:01 -0700 Subject: [PATCH 05/10] convert synthetic population to activitysim format, create subarea inputs --- .../convert_urbansim_to_activitysim.ipynb | 1651 +++++++++++++++++ notebooks/prepare_subrea_inputs.ipynb | 258 ++- notebooks/process_landuse.ipynb | 1222 ++++++++++-- 3 files changed, 2907 insertions(+), 224 deletions(-) create mode 100644 notebooks/convert_urbansim_to_activitysim.ipynb diff --git a/notebooks/convert_urbansim_to_activitysim.ipynb b/notebooks/convert_urbansim_to_activitysim.ipynb new file mode 100644 index 0000000..7b43146 --- /dev/null +++ b/notebooks/convert_urbansim_to_activitysim.ipynb @@ -0,0 +1,1651 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\synthetic_population\"\n", + "land_use_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", + "\n", + "block_to_taz_df = pd.read_csv(os.path.join(land_use_dir, \"zonal\", \"shp\", \"taz_2010block_assignment_20230314.csv\"))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## inputs" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "C:\\Users\\USYS671257\\AppData\\Local\\Temp\\ipykernel_14148\\1065152407.py:1: DtypeWarning: Columns (138,139,140,172,173) have mixed types. Specify dtype option on import or set low_memory=False.\n", + " urbansim_pop = pd.read_csv(\n" + ] + } + ], + "source": [ + "urbansim_pop = pd.read_csv(\n", + " os.path.join(input_dir, \"urbansim_population_complete_PUMS_records_run_s64-s102_2019.csv\")\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## households" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hid", + "rawType": "object", + "type": "string" + }, + { + "name": "block_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hh_inc", + "rawType": "int64", + "type": "integer" + }, + { + "name": "workers", + "rawType": "int64", + "type": "integer" + }, + { + "name": "HHsize", + "rawType": "int64", + "type": "integer" + }, + { + "name": "pid", + "rawType": "object", + "type": "string" + }, + { + "name": "VEH", + "rawType": "float64", + "type": "float" + }, + { + "name": "age", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "0f97b1ea-e635-46f7-bef3-ea70e5850c5c", + "rows": [ + [ + "0", + "2009000000007_1", + "250214175021007", + "36590", + "1", + "1", + "2009000000007_1_1", + "0.0", + "43" + ], + [ + "1", + "2009000000007_2", + "250214179021003", + "36590", + "1", + "1", + "2009000000007_2_1", + "0.0", + "43" + ], + [ + "2", + "2009000000007_3", + "250214180031005", + "36590", + "1", + "1", + "2009000000007_3_1", + "0.0", + "43" + ], + [ + "3", + "2009000000064_10", + "250214131004012", + "120076", + "2", + "3", + "2009000000064_10_1", + "2.0", + "42" + ], + [ + "4", + "2009000000064_10", + "250214131004012", + "120076", + "2", + "3", + "2009000000064_10_2", + "2.0", + "44" + ], + [ + "5", + "2009000000064_10", + "250214131004012", + "120076", + "2", + "3", + "2009000000064_10_3", + "2.0", + "14" + ], + [ + "6", + "2009000000064_11", + "250214131004012", + "61456", + "1", + "2", + "2009000000064_11_1", + "2.0", + "42" + ], + [ + "7", + "2009000000064_11", + "250214131004012", + "61456", + "1", + "2", + "2009000000064_11_2", + "2.0", + "14" + ], + [ + "8", + "2009000000064_12", + "250214131005019", + "61456", + "1", + "2", + "2009000000064_12_1", + "2.0", + "42" + ], + [ + "9", + "2009000000064_12", + "250214131005019", + "61456", + "1", + "2", + "2009000000064_12_2", + "2.0", + "14" + ], + [ + "10", + "2009000000064_13", + "250214132002001", + "120076", + "2", + "3", + "2009000000064_13_1", + "2.0", + "42" + ], + [ + "11", + "2009000000064_13", + "250214132002001", + "120076", + "2", + "3", + "2009000000064_13_2", + "2.0", + "44" + ], + [ + "12", + "2009000000064_13", + "250214132002001", + "120076", + "2", + "3", + "2009000000064_13_3", + "2.0", + "14" + ], + [ + "13", + "2009000000064_14", + "250214132004013", + "120076", + "2", + "3", + "2009000000064_14_1", + "2.0", + "42" + ], + [ + "14", + "2009000000064_14", + "250214132004013", + "120076", + "2", + "3", + "2009000000064_14_2", + "2.0", + "44" + ], + [ + "15", + "2009000000064_14", + "250214132004013", + "120076", + "2", + "3", + "2009000000064_14_3", + "2.0", + "14" + ], + [ + "16", + "2009000000064_15", + "250214133003017", + "120076", + "2", + "3", + "2009000000064_15_1", + "2.0", + "42" + ], + [ + "17", + "2009000000064_15", + "250214133003017", + "120076", + "2", + "3", + "2009000000064_15_2", + "2.0", + "44" + ], + [ + "18", + "2009000000064_15", + "250214133003017", + "120076", + "2", + "3", + "2009000000064_15_3", + "2.0", + "14" + ], + [ + "19", + "2009000000064_16", + "250214133003020", + "120076", + "2", + "3", + "2009000000064_16_1", + "2.0", + "42" + ], + [ + "20", + "2009000000064_16", + "250214133003020", + "120076", + "2", + "3", + "2009000000064_16_2", + "2.0", + "44" + ], + [ + "21", + "2009000000064_16", + "250214133003020", + "120076", + "2", + "3", + "2009000000064_16_3", + "2.0", + "14" + ], + [ + "22", + "2009000000064_17", + "250214175023008", + "61456", + "1", + "2", + "2009000000064_17_1", + "2.0", + "42" + ], + [ + "23", + "2009000000064_17", + "250214175023008", + "61456", + "1", + "2", + "2009000000064_17_2", + "2.0", + "14" + ], + [ + "24", + "2009000000064_18", + "250214412042023", + "120076", + "2", + "3", + "2009000000064_18_1", + "2.0", + "42" + ], + [ + "25", + "2009000000064_18", + "250214412042023", + "120076", + "2", + "3", + "2009000000064_18_2", + "2.0", + "44" + ], + [ + "26", + "2009000000064_18", + "250214412042023", + "120076", + "2", + "3", + "2009000000064_18_3", + "2.0", + "14" + ], + [ + "27", + "2009000000064_19", + "250250104032003", + "61456", + "1", + "2", + "2009000000064_19_1", + "2.0", + "42" + ], + [ + "28", + "2009000000064_19", + "250250104032003", + "61456", + "1", + "2", + "2009000000064_19_2", + "2.0", + "14" + ], + [ + "29", + "2009000000064_4", + "250173504002003", + "120076", + "2", + "3", + "2009000000064_4_1", + "2.0", + "42" + ], + [ + "30", + "2009000000064_4", + "250173504002003", + "120076", + "2", + "3", + "2009000000064_4_2", + "2.0", + "44" + ], + [ + "31", + "2009000000064_4", + "250173504002003", + "120076", + "2", + "3", + "2009000000064_4_3", + "2.0", + "14" + ], + [ + "32", + "2009000000064_5", + "250214112002000", + "120076", + "2", + "3", + "2009000000064_5_1", + "2.0", + "42" + ], + [ + "33", + "2009000000064_5", + "250214112002000", + "120076", + "2", + "3", + "2009000000064_5_2", + "2.0", + "44" + ], + [ + "34", + "2009000000064_5", + "250214112002000", + "120076", + "2", + "3", + "2009000000064_5_3", + "2.0", + "14" + ], + [ + "35", + "2009000000064_6", + "250214123002013", + "61456", + "1", + "2", + "2009000000064_6_1", + "2.0", + "42" + ], + [ + "36", + "2009000000064_6", + "250214123002013", + "61456", + "1", + "2", + "2009000000064_6_2", + "2.0", + "14" + ], + [ + "37", + "2009000000064_7", + "250214123004013", + "61456", + "1", + "2", + "2009000000064_7_1", + "2.0", + "42" + ], + [ + "38", + "2009000000064_7", + "250214123004013", + "61456", + "1", + "2", + "2009000000064_7_2", + "2.0", + "14" + ], + [ + "39", + "2009000000064_8", + "250214131001053", + "120076", + "2", + "3", + "2009000000064_8_1", + "2.0", + "42" + ], + [ + "40", + "2009000000064_8", + "250214131001053", + "120076", + "2", + "3", + "2009000000064_8_2", + "2.0", + "44" + ], + [ + "41", + "2009000000064_8", + "250214131001053", + "120076", + "2", + "3", + "2009000000064_8_3", + "2.0", + "14" + ], + [ + "42", + "2009000000064_9", + "250214131001055", + "120076", + "2", + "3", + "2009000000064_9_1", + "2.0", + "42" + ], + [ + "43", + "2009000000064_9", + "250214131001055", + "120076", + "2", + "3", + "2009000000064_9_2", + "2.0", + "44" + ], + [ + "44", + "2009000000064_9", + "250214131001055", + "120076", + "2", + "3", + "2009000000064_9_3", + "2.0", + "14" + ], + [ + "45", + "2009000000393_20", + "250092021024011", + "61456", + "1", + "3", + "2009000000393_20_1", + "2.0", + "49" + ], + [ + "46", + "2009000000393_20", + "250092021024011", + "61456", + "1", + "3", + "2009000000393_20_2", + "2.0", + "20" + ], + [ + "47", + "2009000000393_20", + "250092021024011", + "61456", + "1", + "3", + "2009000000393_20_3", + "2.0", + "15" + ], + [ + "48", + "2009000000393_21", + "250092022001007", + "61456", + "1", + "3", + "2009000000393_21_1", + "2.0", + "49" + ], + [ + "49", + "2009000000393_21", + "250092022001007", + "61456", + "1", + "3", + "2009000000393_21_2", + "2.0", + "20" + ] + ], + "shape": { + "columns": 8, + "rows": 6784477 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
hidblock_idhh_incworkersHHsizepidVEHage
02009000000007_125021417502100736590112009000000007_1_10.043
12009000000007_225021417902100336590112009000000007_2_10.043
22009000000007_325021418003100536590112009000000007_3_10.043
32009000000064_10250214131004012120076232009000000064_10_12.042
42009000000064_10250214131004012120076232009000000064_10_22.044
...........................
67844722013001492699_2756389250092662003000101828332013001492699_2756389_33.029
67844732013001492699_275639025009266400300626190222013001492699_2756390_13.064
67844742013001492699_275639025009266400300626190222013001492699_2756390_23.029
67844752013001492699_275639125009268200301426190222013001492699_2756391_13.064
67844762013001492699_275639125009268200301426190222013001492699_2756391_23.029
\n", + "

6784477 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " hid block_id hh_inc workers HHsize \\\n", + "0 2009000000007_1 250214175021007 36590 1 1 \n", + "1 2009000000007_2 250214179021003 36590 1 1 \n", + "2 2009000000007_3 250214180031005 36590 1 1 \n", + "3 2009000000064_10 250214131004012 120076 2 3 \n", + "4 2009000000064_10 250214131004012 120076 2 3 \n", + "... ... ... ... ... ... \n", + "6784472 2013001492699_2756389 250092662003000 101828 3 3 \n", + "6784473 2013001492699_2756390 250092664003006 26190 2 2 \n", + "6784474 2013001492699_2756390 250092664003006 26190 2 2 \n", + "6784475 2013001492699_2756391 250092682003014 26190 2 2 \n", + "6784476 2013001492699_2756391 250092682003014 26190 2 2 \n", + "\n", + " pid VEH age \n", + "0 2009000000007_1_1 0.0 43 \n", + "1 2009000000007_2_1 0.0 43 \n", + "2 2009000000007_3_1 0.0 43 \n", + "3 2009000000064_10_1 2.0 42 \n", + "4 2009000000064_10_2 2.0 44 \n", + "... ... ... ... \n", + "6784472 2013001492699_2756389_3 3.0 29 \n", + "6784473 2013001492699_2756390_1 3.0 64 \n", + "6784474 2013001492699_2756390_2 3.0 29 \n", + "6784475 2013001492699_2756391_1 3.0 64 \n", + "6784476 2013001492699_2756391_2 3.0 29 \n", + "\n", + "[6784477 rows x 8 columns]" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "urbansim_pop[[\"hid\",\"block_id\",\"hh_inc\",\"workers\",\"HHsize\",\"pid\",\"VEH\",\"age\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2756405 households\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "household_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "block_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "income", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hhsize", + "rawType": "int64", + "type": "integer" + }, + { + "name": "HHT", + "rawType": "int64", + "type": "integer" + }, + { + "name": "auto_ownership", + "rawType": "float64", + "type": "float" + }, + { + "name": "num_workers", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "364781e4-f7ee-489c-8c44-53936c0d3248", + "rows": [ + [ + "0", + "1", + "250214175021007", + "36590", + "1", + "5", + "0.0", + "1" + ], + [ + "1", + "2", + "250214179021003", + "36590", + "1", + "5", + "0.0", + "1" + ], + [ + "2", + "3", + "250214180031005", + "36590", + "1", + "5", + "0.0", + "1" + ], + [ + "3", + "4", + "250138131021025", + "15128", + "3", + "1", + "1.0", + "0" + ], + [ + "4", + "5", + "250138131022010", + "15128", + "4", + "1", + "1.0", + "0" + ] + ], + "shape": { + "columns": 7, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
household_idblock_idincomehhsizeHHTauto_ownershipnum_workers
0125021417502100736590150.01
1225021417902100336590150.01
2325021418003100536590150.01
3425013813102102515128311.00
4525013813102201015128411.00
\n", + "
" + ], + "text/plain": [ + " household_id block_id income hhsize HHT auto_ownership \\\n", + "0 1 250214175021007 36590 1 5 0.0 \n", + "1 2 250214179021003 36590 1 5 0.0 \n", + "2 3 250214180031005 36590 1 5 0.0 \n", + "3 4 250138131021025 15128 3 1 1.0 \n", + "4 5 250138131022010 15128 4 1 1.0 \n", + "\n", + " num_workers \n", + "0 1 \n", + "1 1 \n", + "2 1 \n", + "3 0 \n", + "4 0 " + ] + }, + "execution_count": 112, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hh = (\n", + " urbansim_pop.groupby(\"hid\")\n", + " .agg(\n", + " block_id = (\"block_id\", \"first\"),\n", + " income = (\"hh_inc\", \"first\"),\n", + " hhsize = (\"HHsize\", \"first\"),\n", + " num_workers = (\"workers\", \"first\"),\n", + " HHT = (\"HHtype\", \"first\"),\n", + " auto_ownership = (\"VEH\", \"max\")\n", + " )\n", + " .reset_index()\n", + " .rename(columns={\"hid\": \"hid_orig\"})\n", + ")\n", + "\n", + "hh[\"household_id\"] = np.arange(1, len(hh) + 1)\n", + "hid_xwalk = dict(zip(hh[\"hid_orig\"], hh[\"household_id\"]))\n", + "\n", + "hh = hh[[\"household_id\", \"block_id\", \"income\", \"hhsize\", \"HHT\", \"auto_ownership\", \"num_workers\"]]\n", + "\n", + "print(f\"{len(hh)} households\")\n", + "hh.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## persons" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "metadata": {}, + "outputs": [], + "source": [ + "ptype_labels = {\n", + " 1: \"Full-time worker\",\n", + " 2: \"Part-time worker\",\n", + " 3: \"College student\",\n", + " 4: \"Non-working adult\",\n", + " 5: \"Retired\",\n", + " 6: \"Driving-age student\",\n", + " 7: \"Non-driving student\",\n", + " 8: \"Child too young for school\",\n", + "}\n", + "pemploy_labels = {\n", + " 1: \"Full-time worker\", \n", + " 2: \"Part-time worker\", \n", + " 3: \"Not employed\", \n", + " 4: \"Students under 16\"\n", + "}\n", + "pstudent_labels = {\n", + " 1: \"Preschool through grade 12 student\", \n", + " 2: \"University/Professional school student\", \n", + " 3: \"Non student\"\n", + "}\n", + "\n", + "MINIMUM_SCHOOL_AGE = 6\n", + "MINIMUM_DRIVING_AGE = 16" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [], + "source": [ + "per = urbansim_pop.copy()\n", + "per[\"household_id\"] = per[\"hid\"].map(hid_xwalk)\n", + "per = per.sort_values([\"household_id\", \"pid\"]).reset_index(drop=True)\n", + "per[\"person_id\"] = np.arange(1, len(per) + 1)" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [], + "source": [ + "# pemploy\n", + "per['pemploy'] = 0\n", + "\n", + "is_adult = per['age'] >= 16\n", + "is_child = per['age'] < 16\n", + "is_employed = per['ESR'].isin([1, 2, 4, 5])\n", + "is_not_employed = per['ESR'].isin([3, 6])\n", + "\n", + "per.loc[is_adult & is_employed & (per['WKHP'] >= 35) & (per['WKW'].isin([1,2,3,4])),'pemploy'] = 1\n", + "per.loc[is_adult & is_employed & (per['pemploy'] == 0), 'pemploy'] = 2\n", + "per.loc[is_adult & is_not_employed & (per['pemploy'] == 0), 'pemploy'] = 3\n", + "per.loc[is_child & (per['pemploy'] == 0), 'pemploy'] = 4\n", + "per.loc[(per['pemploy'] == 0), 'pemploy'] = 3 # default 3=not employed" + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "metadata": {}, + "outputs": [], + "source": [ + "def grade(x):\n", + "\tif x<3:\n", + "\t\treturn x\n", + "\telif x in range(3,7,1): #Grade 1 to 4\n", + "\t\treturn 3\n", + "\telif x in range(7,11,1): #Grade 5 to 8\n", + "\t\treturn 4\n", + "\telif x in range(11,15,1): #Grade 9 to 12\n", + "\t\treturn 5\n", + "\telif x==15: #grade 9\n", + "\t\treturn 6\n", + "\telif x==16: #grade 9\n", + "\t\treturn 7\n", + "\telse:\n", + "\t\treturn 0" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "metadata": {}, + "outputs": [], + "source": [ + "# pstudent\n", + "per['GRADE'] = per['SCHG'].apply(grade)\n", + "per['pstudent'] = 3\n", + "per.loc[per['GRADE'].isin([6, 7]), 'pstudent'] = 2\n", + "per.loc[(per['age'] < 20) & (per['GRADE'].isin([1, 2, 3, 4, 5])), 'pstudent'] = 1\n", + "per.loc[(per['age'] < 16) & (~per['GRADE'].isin([6, 7])), 'pstudent'] = 1\n" + ] + }, + { + "cell_type": "code", + "execution_count": 118, + "metadata": {}, + "outputs": [], + "source": [ + "# ptype\n", + "per['ptype'] = 0\n", + "per.loc[(per['age'] < 6), 'ptype'] = 8\n", + "per.loc[(per['age'].between(6, 15)) & (per['pstudent'] == 1), 'ptype'] = 7\n", + "per.loc[(per['age'].between(16, 19)) & (per['pstudent'] == 1), 'ptype'] = 6\n", + "# per.loc[(per['age'] >= 16) & (per['pstudent'] == 2), 'ptype'] = 3\n", + "per.loc[(per['pstudent'] == 2), 'ptype'] = 3\n", + "per.loc[(per['age'] >= 16) & (per['pemploy'] == 1) & (per['pstudent'] == 3), 'ptype'] = 1\n", + "per.loc[(per['age'] >= 16) & (per['pemploy'] == 2) & (per['pstudent'] == 3), 'ptype'] = 2\n", + "per.loc[(per['age'].between(16, 64)) & (per['pemploy'] == 3) & (per['pstudent'] == 3),'ptype'] = 4\n", + "per.loc[(per['age'] >= 65) & (per['pemploy'] == 3) & (per['pstudent'] == 3), 'ptype'] = 5\n", + "# per.loc[(per['pemploy'] == 0), 'pemploy'] = 3 # default 3=not employed" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6784477 persons\n" + ] + }, + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "person_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "household_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "PNUM", + "rawType": "int64", + "type": "integer" + }, + { + "name": "age", + "rawType": "int64", + "type": "integer" + }, + { + "name": "sex", + "rawType": "float64", + "type": "float" + }, + { + "name": "pemploy", + "rawType": "int64", + "type": "integer" + }, + { + "name": "pstudent", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ptype", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "6118d47a-ffcb-4b05-8aa8-8b56600f6c65", + "rows": [ + [ + "0", + "1", + "1", + "1", + "43", + "2.0", + "2", + "3", + "2" + ], + [ + "1", + "2", + "2", + "1", + "43", + "2.0", + "2", + "3", + "2" + ], + [ + "2", + "3", + "3", + "1", + "43", + "2.0", + "2", + "3", + "2" + ], + [ + "3", + "4", + "4", + "1", + "38", + "2.0", + "3", + "3", + "4" + ], + [ + "4", + "5", + "4", + "2", + "13", + "2.0", + "4", + "1", + "7" + ] + ], + "shape": { + "columns": 8, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
person_idhousehold_idPNUMagesexpemploypstudentptype
0111432.0232
1221432.0232
2331432.0232
3441382.0334
4542132.0417
\n", + "
" + ], + "text/plain": [ + " person_id household_id PNUM age sex pemploy pstudent ptype\n", + "0 1 1 1 43 2.0 2 3 2\n", + "1 2 2 1 43 2.0 2 3 2\n", + "2 3 3 1 43 2.0 2 3 2\n", + "3 4 4 1 38 2.0 3 3 4\n", + "4 5 4 2 13 2.0 4 1 7" + ] + }, + "execution_count": 119, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "per = per[[\"person_id\", \"household_id\", \"person_num\",\n", + " \"age\", \"SEX\", \"pemploy\", \"pstudent\", \"ptype\"]].rename(\n", + " columns={\n", + " \"person_num\": \"PNUM\",\n", + " \"SEX\": \"sex\"\n", + " }\n", + " )\n", + "\n", + "print(f\"{len(per)} persons\")\n", + "per.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## distribution" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " count percent\n", + "ptype \n", + "Full-time worker 2315798 34.13\n", + "Part-time worker 686766 10.12\n", + "College student 453852 6.69\n", + "Non-working adult 1060625 15.63\n", + "Retired 884012 13.03\n", + "Driving-age student 177565 2.62\n", + "Non-driving student 762015 11.23\n", + "Child too young for school 443844 6.54\n", + "\n", + " count percent\n", + "pemploy \n", + "Full-time worker 2461894 36.29\n", + "Part-time worker 885106 13.05\n", + "Not employed 2224479 32.79\n", + "Students under 16 1212998 17.88\n", + "\n", + " count percent\n", + "pstudent \n", + "Preschool through grade 12 student 1383424 20.39\n", + "University/Professional school student 453852 6.69\n", + "Non student 4947201 72.92\n" + ] + } + ], + "source": [ + "for col, labels in [(\"ptype\", ptype_labels), (\"pemploy\", pemploy_labels), (\"pstudent\", pstudent_labels)]:\n", + " counts = per[col].value_counts().sort_index()\n", + " summary = pd.DataFrame(\n", + " {\n", + " \"count\": counts, \n", + " \"percent\": round(counts / len(per) * 100, 2)\n", + " }\n", + " )\n", + " summary.index = summary.index.map(labels)\n", + " print(f\"\\n{summary.to_string()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [], + "source": [ + "block_to_taz_df = block_to_taz_df.rename(columns={\"taz_id\":\"TAZ\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": {}, + "outputs": [], + "source": [ + "hh = hh.merge(\n", + " block_to_taz_df, \n", + " on = \"block_id\", \n", + " how = \"left\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 123, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6784477" + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(urbansim_pop)" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "6784477" + ] + }, + "execution_count": 124, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(per)" + ] + }, + { + "cell_type": "code", + "execution_count": 125, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(6784477)" + ] + }, + "execution_count": 125, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hh.hhsize.sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": {}, + "outputs": [], + "source": [ + "hh.to_csv(os.path.join(input_dir, \"processed\", \"households.csv\"), index=False)\n", + "per.to_csv(os.path.join(input_dir, \"processed\", \"persons.csv\"), index=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "myenv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.19" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/prepare_subrea_inputs.ipynb b/notebooks/prepare_subrea_inputs.ipynb index eed73cc..c14a3ee 100644 --- a/notebooks/prepare_subrea_inputs.ipynb +++ b/notebooks/prepare_subrea_inputs.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -20,21 +20,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "landuse_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", + "population_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\synthetic_population\"\n", "skim_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\"" ] }, @@ -47,19 +38,9 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "zone_file = gpd.read_file(os.path.join(landuse_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\"))\n", "landuse_file = pd.read_csv(os.path.join(landuse_dir, \"processed\", \"land_use.csv\"))" @@ -67,61 +48,76 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "downtown_taz_ids = zone_file[zone_file[\"ring\"].isin([0,1])].taz_id.to_list()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "landuse_file = landuse_file[landuse_file[\"TAZ\"].isin(downtown_taz_ids)]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "landuse_file.to_csv(os.path.join(landuse_dir, \"processed\", \"land_use_subarea.csv\"), index=False)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## subarea population" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "hh_df = pd.read_csv(os.path.join(population_dir, \"processed\", \"households.csv\"))\n", + "pop_df = pd.read_csv(os.path.join(population_dir, \"processed\", \"persons.csv\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "hh_df = hh_df[hh_df[\"TAZ\"].isin(downtown_taz_ids)]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "pop_df = pop_df[pop_df[\"household_id\"].isin(hh_df[\"household_id\"])]" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "hh_df.to_csv(os.path.join(population_dir, \"processed\", \"households_subarea.csv\"), index=False)\n", + "pop_df.to_csv(os.path.join(population_dir, \"processed\", \"persons_subarea.csv\"), index=False)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -131,19 +127,9 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [ - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], + "outputs": [], "source": [ "skims = [\n", " \"hwy_am_pm\",\n", @@ -160,16 +146,128 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [ { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mnotebook controller is DISPOSED. \n", - "\u001b[1;31mView Jupyter log for further details." + "name": "stdout", + "output_type": "stream", + "text": [ + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_am_pm.omx\n", + "save matrix da_time__AM\n", + "save matrix da_time__PM\n", + "save matrix da_toll__AM\n", + "save matrix da_toll__PM\n", + "save matrix dist__AM\n", + "save matrix dist__PM\n", + "save matrix rs_fare__AM\n", + "save matrix rs_fare__PM\n", + "save matrix sr_time__AM\n", + "save matrix sr_time__PM\n", + "save matrix sr_toll__AM\n", + "save matrix sr_toll__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_md_ev.omx\n", + "save matrix da_time__EV\n", + "save matrix da_time__MD\n", + "save matrix da_toll__EV\n", + "save matrix da_toll__MD\n", + "save matrix dist__EV\n", + "save matrix dist__MD\n", + "save matrix rs_fare__EV\n", + "save matrix rs_fare__MD\n", + "save matrix sr_time__EV\n", + "save matrix sr_time__MD\n", + "save matrix sr_toll__EV\n", + "save matrix sr_toll__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\nm_daily.omx\n", + "save matrix dist_nm\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\nm_daily_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_am_pm.omx\n", + "save matrix tw_fare__AM\n", + "save matrix tw_fare__PM\n", + "save matrix tw_gen_cost__AM\n", + "save matrix tw_gen_cost__PM\n", + "save matrix tw_ivtt__AM\n", + "save matrix tw_ivtt__PM\n", + "save matrix tw_iwait__AM\n", + "save matrix tw_iwait__PM\n", + "save matrix tw_tdist__AM\n", + "save matrix tw_tdist__PM\n", + "save matrix tw_walk__AM\n", + "save matrix tw_walk__PM\n", + "save matrix tw_xfer__AM\n", + "save matrix tw_xfer__PM\n", + "save matrix tw_xwait__AM\n", + "save matrix tw_xwait__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_md_ev.omx\n", + "save matrix tw_fare__EV\n", + "save matrix tw_fare__MD\n", + "save matrix tw_gen_cost__EV\n", + "save matrix tw_gen_cost__MD\n", + "save matrix tw_ivtt__EV\n", + "save matrix tw_ivtt__MD\n", + "save matrix tw_iwait__EV\n", + "save matrix tw_iwait__MD\n", + "save matrix tw_tdist__EV\n", + "save matrix tw_tdist__MD\n", + "save matrix tw_walk__EV\n", + "save matrix tw_walk__MD\n", + "save matrix tw_xfer__EV\n", + "save matrix tw_xfer__MD\n", + "save matrix tw_xwait__EV\n", + "save matrix tw_xwait__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_am_pm.omx\n", + "save matrix ta_auto_cost__AM\n", + "save matrix ta_auto_cost__PM\n", + "save matrix ta_ddist__AM\n", + "save matrix ta_ddist__PM\n", + "save matrix ta_dtime__AM\n", + "save matrix ta_dtime__PM\n", + "save matrix ta_fare__AM\n", + "save matrix ta_fare__PM\n", + "save matrix ta_gen_cost__AM\n", + "save matrix ta_gen_cost__PM\n", + "save matrix ta_ivtt__AM\n", + "save matrix ta_ivtt__PM\n", + "save matrix ta_iwait__AM\n", + "save matrix ta_iwait__PM\n", + "save matrix ta_tdist__AM\n", + "save matrix ta_tdist__PM\n", + "save matrix ta_walk__AM\n", + "save matrix ta_walk__PM\n", + "save matrix ta_xfer__AM\n", + "save matrix ta_xfer__PM\n", + "save matrix ta_xwait__AM\n", + "save matrix ta_xwait__PM\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_md_ev.omx\n", + "save matrix ta_auto_cost__EV\n", + "save matrix ta_auto_cost__MD\n", + "save matrix ta_ddist__EV\n", + "save matrix ta_ddist__MD\n", + "save matrix ta_dtime__EV\n", + "save matrix ta_dtime__MD\n", + "save matrix ta_fare__EV\n", + "save matrix ta_fare__MD\n", + "save matrix ta_gen_cost__EV\n", + "save matrix ta_gen_cost__MD\n", + "save matrix ta_ivtt__EV\n", + "save matrix ta_ivtt__MD\n", + "save matrix ta_iwait__EV\n", + "save matrix ta_iwait__MD\n", + "save matrix ta_tdist__EV\n", + "save matrix ta_tdist__MD\n", + "save matrix ta_walk__EV\n", + "save matrix ta_walk__MD\n", + "save matrix ta_xfer__EV\n", + "save matrix ta_xfer__MD\n", + "save matrix ta_xwait__EV\n", + "save matrix ta_xwait__MD\n", + "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_md_ev_subarea.omx\n" ] } ], diff --git a/notebooks/process_landuse.ipynb b/notebooks/process_landuse.ipynb index 8cecc2c..1e28510 100644 --- a/notebooks/process_landuse.ipynb +++ b/notebooks/process_landuse.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 14, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -32,11 +32,11 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "block_to_taz_file = os.path.join(input_dir, \"zonal\", \"shp\", \"taz_2010block_allocation_20230314.csv\")\n", + "block_to_taz_file = os.path.join(input_dir, \"zonal\", \"shp\", \"taz_2010block_assignment_20230314.csv\")\n", "\n", "# zone shape file\n", "zone_shape_file = os.path.join(input_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\")\n", @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -139,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -148,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -176,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -214,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -233,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -246,7 +246,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -256,7 +256,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -319,7 +319,7 @@ "type": "integer" } ], - "ref": "9b342455-ca72-417d-b207-71bbedf25be8", + "ref": "c30e7556-7b32-4a1f-b4c4-c3936ce9f710", "rows": [ [ "0", @@ -1187,7 +1187,7 @@ "[5839 rows x 10 columns]" ] }, - "execution_count": 26, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1205,7 +1205,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -1215,10 +1215,6 @@ " employment_columns = [col for col in merged.columns \n", " if col not in ['block_id', 'taz_id', 'area_fct']]\n", "\n", - " # apply area factor\n", - " for col in employment_columns:\n", - " merged[col] = merged[col] * merged['area_fct']\n", - "\n", " taz_employment = merged.groupby('taz_id')[employment_columns].sum().reset_index()\n", "\n", " taz_employment['RETEMPN'] = taz_employment['6_ret_leis']\n", @@ -1242,7 +1238,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -1257,7 +1253,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -1273,128 +1269,1066 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def process_population_data(population_df, block_to_taz_df):\n", - " merged = population_df.merge(block_to_taz_df, on='block_id', how='left')\n", - "\n", - " merged['is_age_0519'] = ((merged['age'] >= 5) & (merged['age'] <= 19)).astype(int)\n", - "\n", - " # apply area factor\n", - " merged['pop_weighted'] = merged['area_fct']\n", - " merged['pop_age0519_weighted'] = merged['area_fct'] * merged['is_age_0519']\n", - " \n", - " taz_population = merged.groupby('taz_id').agg(\n", - " TOTPOP=('pop_weighted', 'sum'),\n", - " AGE0519=('pop_age0519_weighted', 'sum')\n", - " ).reset_index()\n", - " \n", - " taz_population = taz_population.rename(\n", - " columns={'taz_id': 'TAZ'}\n", - " )\n", - " taz_population = taz_population.sort_values('TAZ').reset_index(drop=True)\n", - "\n", - " return taz_population" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [], - "source": [ - "ma_pop_agg = process_population_data(ma_pop_df, block_to_taz_df)\n", - "nhri_pop_agg = process_population_data(nhri_pop_df, block_to_taz_df)\n", - "\n", - "pop_combined = pd.concat([ma_pop_agg, nhri_pop_agg], ignore_index=False)\n", - "pop_combined = pop_combined.set_index('TAZ')\n", - "pop_combined = pop_combined.groupby(level=0).sum().reset_index()\n", - "pop_combined = pop_combined.sort_values('TAZ').reset_index(drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "zone_shape_df = zone_shape_df.merge(pop_combined, on='TAZ', how='left').fillna(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## parking" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "zone_shape_df = zone_shape_df.merge(\n", - " parking_df[['taz_id','cost_hr']].rename(columns={'taz_id': 'TAZ'}),\n", - " on='TAZ', \n", - " how='left').fillna(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "# TODO: update parking cost\n", - "zone_shape_df.rename(columns={'cost_hr': 'PRKCST'}, inplace=True)\n", - "zone_shape_df['OPRKCST'] = zone_shape_df['PRKCST']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## enrollment" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [], - "source": [ - "enroll_df['HSENROLL'] = enroll_df['k12']\n", - "\n", - "# TODO: update with actual enrollment data\n", - "enroll_df['COLLFTE'] = (enroll_df['college_total'] * 0.9).astype(int)\n", - "enroll_df['COLLPTE'] = (enroll_df['college_total'] * 0.1).astype(int)" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "zone_shape_df = zone_shape_df.merge(\n", - " enroll_df[['taz_id','HSENROLL', 'COLLFTE', 'COLLPTE']].rename(columns={'taz_id': 'TAZ'}),\n", - " on='TAZ', \n", - " how='left').fillna(0)" - ] - }, - { - "cell_type": "markdown", + "execution_count": 17, "metadata": {}, - "source": [ - "## output" - ] - }, - { - "cell_type": "code", - "execution_count": 37, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hid", + "rawType": "object", + "type": "string" + }, + { + "name": "block_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hh_inc", + "rawType": "int64", + "type": "integer" + }, + { + "name": "persons", + "rawType": "int64", + "type": "integer" + }, + { + "name": "workers", + "rawType": "int64", + "type": "integer" + }, + { + "name": "children", + "rawType": "int64", + "type": "integer" + }, + { + "name": "person_num", + "rawType": "int64", + "type": "integer" + }, + { + "name": "age", + "rawType": "int64", + "type": "integer" + }, + { + "name": "is_worker", + "rawType": "int64", + "type": "integer" + }, + { + "name": "wage_inc", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "a9bb9b5e-3f82-4135-8134-4b8c5724edb1", + "rows": [ + [ + "0", + "2009000000007_1", + "250214175021007", + "36590", + "1", + "1", + "0", + "1", + "43", + "1", + "36590" + ], + [ + "1", + "2009000000007_2", + "250214179021003", + "36590", + "1", + "1", + "0", + "1", + "43", + "1", + "36590" + ], + [ + "2", + "2009000000007_3", + "250214180031005", + "36590", + "1", + "1", + "0", + "1", + "43", + "1", + "36590" + ], + [ + "3", + "2009000000064_10", + "250214123004013", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "4", + "2009000000064_10", + "250214123004013", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "5", + "2009000000064_10", + "250214123004013", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "6", + "2009000000064_11", + "250214131001053", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "7", + "2009000000064_11", + "250214131001053", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "8", + "2009000000064_12", + "250214131001055", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "9", + "2009000000064_12", + "250214131001055", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "10", + "2009000000064_13", + "250214131004012", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "11", + "2009000000064_13", + "250214131004012", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "12", + "2009000000064_13", + "250214131004012", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "13", + "2009000000064_14", + "250214131004012", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "14", + "2009000000064_14", + "250214131004012", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "15", + "2009000000064_15", + "250214131005019", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "16", + "2009000000064_15", + "250214131005019", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "17", + "2009000000064_15", + "250214131005019", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "18", + "2009000000064_16", + "250214132002001", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "19", + "2009000000064_16", + "250214132002001", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "20", + "2009000000064_16", + "250214132002001", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "21", + "2009000000064_17", + "250214132004013", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "22", + "2009000000064_17", + "250214132004013", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "23", + "2009000000064_18", + "250214133003017", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "24", + "2009000000064_18", + "250214133003017", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "25", + "2009000000064_18", + "250214133003017", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "26", + "2009000000064_19", + "250214133003020", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "27", + "2009000000064_19", + "250214133003020", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "28", + "2009000000064_4", + "250173507006005", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "29", + "2009000000064_4", + "250173507006005", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "30", + "2009000000064_5", + "250173732001012", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "31", + "2009000000064_5", + "250173732001012", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "32", + "2009000000064_6", + "250173861002039", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "33", + "2009000000064_6", + "250173861002039", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "34", + "2009000000064_6", + "250173861002039", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "35", + "2009000000064_7", + "250214021014019", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "36", + "2009000000064_7", + "250214021014019", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "37", + "2009000000064_7", + "250214021014019", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "38", + "2009000000064_8", + "250214112002000", + "61456", + "2", + "1", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "39", + "2009000000064_8", + "250214112002000", + "61456", + "2", + "1", + "1", + "2", + "14", + "0", + "0" + ], + [ + "40", + "2009000000064_9", + "250214123002013", + "120076", + "3", + "2", + "1", + "1", + "42", + "1", + "61456" + ], + [ + "41", + "2009000000064_9", + "250214123002013", + "120076", + "3", + "2", + "1", + "2", + "44", + "1", + "58620" + ], + [ + "42", + "2009000000064_9", + "250214123002013", + "120076", + "3", + "2", + "1", + "3", + "14", + "0", + "0" + ], + [ + "43", + "2009000000393_20", + "250092021024011", + "61456", + "3", + "1", + "1", + "1", + "49", + "1", + "61456" + ], + [ + "44", + "2009000000393_20", + "250092021024011", + "61456", + "3", + "1", + "1", + "2", + "20", + "0", + "0" + ], + [ + "45", + "2009000000393_20", + "250092021024011", + "61456", + "3", + "1", + "1", + "3", + "15", + "0", + "0" + ], + [ + "46", + "2009000000393_21", + "250092022001007", + "61456", + "3", + "1", + "1", + "1", + "49", + "1", + "61456" + ], + [ + "47", + "2009000000393_21", + "250092022001007", + "61456", + "3", + "1", + "1", + "2", + "20", + "0", + "0" + ], + [ + "48", + "2009000000393_21", + "250092022001007", + "61456", + "3", + "1", + "1", + "3", + "15", + "0", + "0" + ], + [ + "49", + "2009000000393_22", + "250092022002034", + "61456", + "3", + "1", + "1", + "1", + "49", + "1", + "61456" + ] + ], + "shape": { + "columns": 10, + "rows": 6784306 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
hidblock_idhh_incpersonsworkerschildrenperson_numageis_workerwage_inc
02009000000007_125021417502100736590110143136590
12009000000007_225021417902100336590110143136590
22009000000007_325021418003100536590110143136590
32009000000064_10250214123004013120076321142161456
42009000000064_10250214123004013120076321244158620
.................................
67843012013001492699_2756389250092683004009101828330164111346
67843022013001492699_2756389250092683004009101828330264175638
67843032013001492699_2756389250092683004009101828330329114844
67843042013001492699_275639025009268300400926190220164111346
67843052013001492699_275639025009268300400926190220229114844
\n", + "

6784306 rows × 10 columns

\n", + "
" + ], + "text/plain": [ + " hid block_id hh_inc persons workers \\\n", + "0 2009000000007_1 250214175021007 36590 1 1 \n", + "1 2009000000007_2 250214179021003 36590 1 1 \n", + "2 2009000000007_3 250214180031005 36590 1 1 \n", + "3 2009000000064_10 250214123004013 120076 3 2 \n", + "4 2009000000064_10 250214123004013 120076 3 2 \n", + "... ... ... ... ... ... \n", + "6784301 2013001492699_2756389 250092683004009 101828 3 3 \n", + "6784302 2013001492699_2756389 250092683004009 101828 3 3 \n", + "6784303 2013001492699_2756389 250092683004009 101828 3 3 \n", + "6784304 2013001492699_2756390 250092683004009 26190 2 2 \n", + "6784305 2013001492699_2756390 250092683004009 26190 2 2 \n", + "\n", + " children person_num age is_worker wage_inc \n", + "0 0 1 43 1 36590 \n", + "1 0 1 43 1 36590 \n", + "2 0 1 43 1 36590 \n", + "3 1 1 42 1 61456 \n", + "4 1 2 44 1 58620 \n", + "... ... ... ... ... ... \n", + "6784301 0 1 64 1 11346 \n", + "6784302 0 2 64 1 75638 \n", + "6784303 0 3 29 1 14844 \n", + "6784304 0 1 64 1 11346 \n", + "6784305 0 2 29 1 14844 \n", + "\n", + "[6784306 rows x 10 columns]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ma_pop_df" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def process_population_data(population_df, block_to_taz_df):\n", + " merged = population_df.merge(block_to_taz_df, on='block_id', how='left')\n", + "\n", + " merged['is_age_0519'] = ((merged['age'] >= 5) & (merged['age'] <= 19)).astype(int)\n", + "\n", + " taz_population = merged.groupby('taz_id').agg(\n", + " TOTPOP=('age', 'count'),\n", + " AGE0519=('is_age_0519', 'sum')\n", + " ).reset_index()\n", + " \n", + " taz_population = taz_population.rename(\n", + " columns={'taz_id': 'TAZ'}\n", + " )\n", + " taz_population = taz_population.sort_values('TAZ').reset_index(drop=True)\n", + "\n", + " return taz_population" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "ma_pop_agg = process_population_data(ma_pop_df, block_to_taz_df)\n", + "nhri_pop_agg = process_population_data(nhri_pop_df, block_to_taz_df)\n", + "\n", + "pop_combined = pd.concat([ma_pop_agg, nhri_pop_agg], ignore_index=False)\n", + "pop_combined = pop_combined.set_index('TAZ')\n", + "pop_combined = pop_combined.groupby(level=0).sum().reset_index()\n", + "pop_combined = pop_combined.sort_values('TAZ').reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(pop_combined, on='TAZ', how='left').fillna(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## parking" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(\n", + " parking_df[['taz_id','cost_hr']].rename(columns={'taz_id': 'TAZ'}),\n", + " on='TAZ', \n", + " how='left').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: update parking cost\n", + "zone_shape_df.rename(columns={'cost_hr': 'PRKCST'}, inplace=True)\n", + "zone_shape_df['OPRKCST'] = zone_shape_df['PRKCST']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## enrollment" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "enroll_df['HSENROLL'] = enroll_df['k12']\n", + "\n", + "# TODO: update with actual enrollment data\n", + "enroll_df['COLLFTE'] = (enroll_df['college_total'] * 0.9).astype(int)\n", + "enroll_df['COLLPTE'] = (enroll_df['college_total'] * 0.1).astype(int)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(\n", + " enroll_df[['taz_id','HSENROLL', 'COLLFTE', 'COLLPTE']].rename(columns={'taz_id': 'TAZ'}),\n", + " on='TAZ', \n", + " how='left').fillna(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## output" + ] + }, + { + "cell_type": "code", + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ From e9421792b090dc5ecdfecef7367cc3fc3ff96c46 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:08:34 -0700 Subject: [PATCH 06/10] fix pytpe issue, update file paths to point to SharePoint --- .../convert_urbansim_to_activitysim.ipynb | 1992 ++++++++++++++++- 1 file changed, 1955 insertions(+), 37 deletions(-) diff --git a/notebooks/convert_urbansim_to_activitysim.ipynb b/notebooks/convert_urbansim_to_activitysim.ipynb index 7b43146..1bf19fd 100644 --- a/notebooks/convert_urbansim_to_activitysim.ipynb +++ b/notebooks/convert_urbansim_to_activitysim.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 108, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -13,14 +13,14 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ - "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\synthetic_population\"\n", - "land_use_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", + "input_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\EXTERNAL\"\n", + "output_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\synthetic_population\"\n", "\n", - "block_to_taz_df = pd.read_csv(os.path.join(land_use_dir, \"zonal\", \"shp\", \"taz_2010block_assignment_20230314.csv\"))\n" + "block_to_taz_df = pd.read_csv(os.path.join(input_dir, \"zonal\", \"shp\", \"taz_2010block_assignment_20230314.csv\"))\n" ] }, { @@ -32,14 +32,14 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\USYS671257\\AppData\\Local\\Temp\\ipykernel_14148\\1065152407.py:1: DtypeWarning: Columns (138,139,140,172,173) have mixed types. Specify dtype option on import or set low_memory=False.\n", + "C:\\Users\\USYS671257\\AppData\\Local\\Temp\\ipykernel_37332\\1065152407.py:1: DtypeWarning: Columns (138,139,140,172,173) have mixed types. Specify dtype option on import or set low_memory=False.\n", " urbansim_pop = pd.read_csv(\n" ] } @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -112,7 +112,7 @@ "type": "integer" } ], - "ref": "0f97b1ea-e635-46f7-bef3-ea70e5850c5c", + "ref": "aa33e846-8984-4088-bd39-1fda36ee2708", "rows": [ [ "0", @@ -856,7 +856,7 @@ "[6784477 rows x 8 columns]" ] }, - "execution_count": 111, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -867,7 +867,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -922,7 +922,7 @@ "type": "integer" } ], - "ref": "364781e4-f7ee-489c-8c44-53936c0d3248", + "ref": "5bea5612-51eb-4a70-9e45-9deefebbf3b7", "rows": [ [ "0", @@ -1079,7 +1079,7 @@ "4 0 " ] }, - "execution_count": 112, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" } @@ -1117,7 +1117,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 80, "metadata": {}, "outputs": [], "source": [ @@ -1149,7 +1149,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 81, "metadata": {}, "outputs": [], "source": [ @@ -1161,7 +1161,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 82, "metadata": {}, "outputs": [], "source": [ @@ -1182,7 +1182,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 83, "metadata": {}, "outputs": [], "source": [ @@ -1205,7 +1205,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -1219,17 +1219,17 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ "# ptype\n", "per['ptype'] = 0\n", + "per.loc[(per['pstudent'] == 2), 'ptype'] = 3\n", "per.loc[(per['age'] < 6), 'ptype'] = 8\n", "per.loc[(per['age'].between(6, 15)) & (per['pstudent'] == 1), 'ptype'] = 7\n", "per.loc[(per['age'].between(16, 19)) & (per['pstudent'] == 1), 'ptype'] = 6\n", "# per.loc[(per['age'] >= 16) & (per['pstudent'] == 2), 'ptype'] = 3\n", - "per.loc[(per['pstudent'] == 2), 'ptype'] = 3\n", "per.loc[(per['age'] >= 16) & (per['pemploy'] == 1) & (per['pstudent'] == 3), 'ptype'] = 1\n", "per.loc[(per['age'] >= 16) & (per['pemploy'] == 2) & (per['pstudent'] == 3), 'ptype'] = 2\n", "per.loc[(per['age'].between(16, 64)) & (per['pemploy'] == 3) & (per['pstudent'] == 3),'ptype'] = 4\n", @@ -1239,7 +1239,1925 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 86, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "bl10_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "muni_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "muni_name", + "rawType": "object", + "type": "string" + }, + { + "name": "rpa_acr", + "rawType": "object", + "type": "string" + }, + { + "name": "mpo", + "rawType": "object", + "type": "string" + }, + { + "name": "hid", + "rawType": "object", + "type": "string" + }, + { + "name": "serialno", + "rawType": "int64", + "type": "integer" + }, + { + "name": "person_num", + "rawType": "int64", + "type": "integer" + }, + { + "name": "block_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "year", + "rawType": "int64", + "type": "integer" + }, + { + "name": "age", + "rawType": "int64", + "type": "integer" + }, + { + "name": "is_worker", + "rawType": "int64", + "type": "integer" + }, + { + "name": "relationship_to_sp1", + "rawType": "int64", + "type": "integer" + }, + { + "name": "pinc2013", + "rawType": "float64", + "type": "float" + }, + { + "name": "hhinc2013", + "rawType": "float64", + "type": "float" + }, + { + "name": "inc_adj", + "rawType": "int64", + "type": "integer" + }, + { + "name": "wage_inc", + "rawType": "int64", + "type": "integer" + }, + { + "name": "hh_inc", + "rawType": "int64", + "type": "integer" + }, + { + "name": "persons", + "rawType": "int64", + "type": "integer" + }, + { + "name": "workers", + "rawType": "int64", + "type": "integer" + }, + { + "name": "tenure", + "rawType": "int64", + "type": "integer" + }, + { + "name": "child", + "rawType": "int64", + "type": "integer" + }, + { + "name": "income_grp", + "rawType": "object", + "type": "string" + }, + { + "name": "children", + "rawType": "int64", + "type": "integer" + }, + { + "name": "adult", + "rawType": "int64", + "type": "integer" + }, + { + "name": "adults", + "rawType": "int64", + "type": "integer" + }, + { + "name": "HHtype", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ageCAT6", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ageHHder", + "rawType": "int64", + "type": "integer" + }, + { + "name": "pid", + "rawType": "object", + "type": "string" + }, + { + "name": "AGEP.x", + "rawType": "int64", + "type": "integer" + }, + { + "name": "SPORDER.x", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ageHH", + "rawType": "float64", + "type": "float" + }, + { + "name": "ht", + "rawType": "float64", + "type": "float" + }, + { + "name": "HHT", + "rawType": "int64", + "type": "integer" + }, + { + "name": "HHsize", + "rawType": "int64", + "type": "integer" + }, + { + "name": "children_in_HH", + "rawType": "int64", + "type": "integer" + }, + { + "name": "SERIALNO", + "rawType": "float64", + "type": "float" + }, + { + "name": "SPORDER.y", + "rawType": "float64", + "type": "float" + }, + { + "name": "AGEP.y", + "rawType": "float64", + "type": "float" + }, + { + "name": "CIT", + "rawType": "float64", + "type": "float" + }, + { + "name": "CITWP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "CITWP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "COW", + "rawType": "float64", + "type": "float" + }, + { + "name": "DDRS", + "rawType": "float64", + "type": "float" + }, + { + "name": "DEAR", + "rawType": "float64", + "type": "float" + }, + { + "name": "DEYE", + "rawType": "float64", + "type": "float" + }, + { + "name": "DOUT", + "rawType": "float64", + "type": "float" + }, + { + "name": "DPHY", + "rawType": "float64", + "type": "float" + }, + { + "name": "DRAT", + "rawType": "float64", + "type": "float" + }, + { + "name": "DRATX", + "rawType": "float64", + "type": "float" + }, + { + "name": "DREM", + "rawType": "float64", + "type": "float" + }, + { + "name": "ENG", + "rawType": "float64", + "type": "float" + }, + { + "name": "FER", + "rawType": "float64", + "type": "float" + }, + { + "name": "GCL", + "rawType": "float64", + "type": "float" + }, + { + "name": "GCM", + "rawType": "float64", + "type": "float" + }, + { + "name": "GCR", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS1", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS2", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS3", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS4", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS5", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS6", + "rawType": "float64", + "type": "float" + }, + { + "name": "HINS7", + "rawType": "float64", + "type": "float" + }, + { + "name": "INTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "JWMNP", + "rawType": "float64", + "type": "float" + }, + { + "name": "JWRIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "JWTR", + "rawType": "float64", + "type": "float" + }, + { + "name": "LANX", + "rawType": "float64", + "type": "float" + }, + { + "name": "MAR", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHD", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHM", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHT", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHW", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHYP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "MARHYP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIG", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIL", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPA", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPB", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPCD", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPE", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPFG", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPH", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPI", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPJ", + "rawType": "float64", + "type": "float" + }, + { + "name": "MLPK", + "rawType": "float64", + "type": "float" + }, + { + "name": "NWAB", + "rawType": "float64", + "type": "float" + }, + { + "name": "NWAV", + "rawType": "float64", + "type": "float" + }, + { + "name": "NWLA", + "rawType": "float64", + "type": "float" + }, + { + "name": "NWLK", + "rawType": "float64", + "type": "float" + }, + { + "name": "NWRE", + "rawType": "float64", + "type": "float" + }, + { + "name": "OIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "PAP", + "rawType": "float64", + "type": "float" + }, + { + "name": "RELP", + "rawType": "float64", + "type": "float" + }, + { + "name": "RETP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SCH", + "rawType": "float64", + "type": "float" + }, + { + "name": "SCHG", + "rawType": "float64", + "type": "float" + }, + { + "name": "SCHL", + "rawType": "float64", + "type": "float" + }, + { + "name": "SEMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SEX", + "rawType": "float64", + "type": "float" + }, + { + "name": "SSIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "WAGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "WKHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "WKL", + "rawType": "float64", + "type": "float" + }, + { + "name": "WKW", + "rawType": "float64", + "type": "float" + }, + { + "name": "WRK", + "rawType": "float64", + "type": "float" + }, + { + "name": "YOEP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "YOEP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "ANC", + "rawType": "float64", + "type": "float" + }, + { + "name": "ANC1P05", + "rawType": "float64", + "type": "float" + }, + { + "name": "ANC1P12", + "rawType": "float64", + "type": "float" + }, + { + "name": "ANC2P05", + "rawType": "float64", + "type": "float" + }, + { + "name": "ANC2P12", + "rawType": "float64", + "type": "float" + }, + { + "name": "DECADE", + "rawType": "float64", + "type": "float" + }, + { + "name": "DIS", + "rawType": "float64", + "type": "float" + }, + { + "name": "DRIVESP", + "rawType": "float64", + "type": "float" + }, + { + "name": "ESP", + "rawType": "float64", + "type": "float" + }, + { + "name": "ESR", + "rawType": "float64", + "type": "float" + }, + { + "name": "FOD1P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FOD2P", + "rawType": "float64", + "type": "float" + }, + { + "name": "HICOV", + "rawType": "float64", + "type": "float" + }, + { + "name": "HISP", + "rawType": "float64", + "type": "float" + }, + { + "name": "INDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "JWAP", + "rawType": "float64", + "type": "float" + }, + { + "name": "JWDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "LANP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "LANP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIGPUMA00", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIGPUMA10", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIGSP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "MIGSP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "MSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "NAICSP", + "rawType": "object", + "type": "string" + }, + { + "name": "NATIVITY", + "rawType": "float64", + "type": "float" + }, + { + "name": "NOP", + "rawType": "float64", + "type": "float" + }, + { + "name": "OC", + "rawType": "float64", + "type": "float" + }, + { + "name": "OCCP02", + "rawType": "object", + "type": "string" + }, + { + "name": "OCCP10", + "rawType": "object", + "type": "string" + }, + { + "name": "OCCP12", + "rawType": "object", + "type": "string" + }, + { + "name": "PAOC", + "rawType": "float64", + "type": "float" + }, + { + "name": "PERNP", + "rawType": "float64", + "type": "float" + }, + { + "name": "PINCP", + "rawType": "float64", + "type": "float" + }, + { + "name": "POBP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "POBP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "POVPIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "POWPUMA00", + "rawType": "float64", + "type": "float" + }, + { + "name": "POWPUMA10", + "rawType": "float64", + "type": "float" + }, + { + "name": "POWSP05", + "rawType": "float64", + "type": "float" + }, + { + "name": "POWSP12", + "rawType": "float64", + "type": "float" + }, + { + "name": "PRIVCOV", + "rawType": "float64", + "type": "float" + }, + { + "name": "PUBCOV", + "rawType": "float64", + "type": "float" + }, + { + "name": "QTRBIR", + "rawType": "float64", + "type": "float" + }, + { + "name": "RAC1P", + "rawType": "float64", + "type": "float" + }, + { + "name": "RAC2P05", + "rawType": "float64", + "type": "float" + }, + { + "name": "RAC2P12", + "rawType": "float64", + "type": "float" + }, + { + "name": "RAC3P05", + "rawType": "float64", + "type": "float" + }, + { + "name": "RAC3P12", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACAIAN", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACASN", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACBLK", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACNHPI", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACNUM", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACSOR", + "rawType": "float64", + "type": "float" + }, + { + "name": "RACWHT", + "rawType": "float64", + "type": "float" + }, + { + "name": "RC", + "rawType": "float64", + "type": "float" + }, + { + "name": "SCIENGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SCIENGRLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SFN", + "rawType": "float64", + "type": "float" + }, + { + "name": "SFR", + "rawType": "float64", + "type": "float" + }, + { + "name": "SOCP00", + "rawType": "object", + "type": "string" + }, + { + "name": "SOCP10", + "rawType": "object", + "type": "string" + }, + { + "name": "SOCP12", + "rawType": "object", + "type": "string" + }, + { + "name": "VPS", + "rawType": "float64", + "type": "float" + }, + { + "name": "WAOB", + "rawType": "float64", + "type": "float" + }, + { + "name": "FAGEP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FANCP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FCITP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FCITWP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FCOWP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDDRSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDEARP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDEYEP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDOUTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDPHYP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDRATP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDRATXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FDREMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FENGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FESRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FFERP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FFODP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FGCLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FGCMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FGCRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS1P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS2P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS3C", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS3P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS4C", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS4P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS5C", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS5P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS6P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHINS7P", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHISP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FINDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FINTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FJWDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FJWMNP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FJWRIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FJWTRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FLANP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FLANXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARHDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARHMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARHTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARHWP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARHYP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMARP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMIGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMIGSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMILPP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMILSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FOCCP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FOIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FPAP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FPOBP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FPOWSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRACP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRELP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRETP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSCHGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSCHLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSCHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSEMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSEXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSSIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWAGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWKHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWKLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWKWP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWRKP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FYOEP", + "rawType": "float64", + "type": "float" + }, + { + "name": "insp", + "rawType": "float64", + "type": "float" + }, + { + "name": "DIVISION", + "rawType": "float64", + "type": "float" + }, + { + "name": "REGION", + "rawType": "float64", + "type": "float" + }, + { + "name": "ST.y", + "rawType": "float64", + "type": "float" + }, + { + "name": "ACR", + "rawType": "float64", + "type": "float" + }, + { + "name": "AGS", + "rawType": "float64", + "type": "float" + }, + { + "name": "BATH", + "rawType": "float64", + "type": "float" + }, + { + "name": "BDSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "BLD", + "rawType": "float64", + "type": "float" + }, + { + "name": "BUS", + "rawType": "float64", + "type": "float" + }, + { + "name": "CONP", + "rawType": "float64", + "type": "float" + }, + { + "name": "ELEP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FS", + "rawType": "float64", + "type": "float" + }, + { + "name": "FULP", + "rawType": "float64", + "type": "float" + }, + { + "name": "GASP", + "rawType": "float64", + "type": "float" + }, + { + "name": "HFL", + "rawType": "float64", + "type": "float" + }, + { + "name": "MHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "MRGI", + "rawType": "float64", + "type": "float" + }, + { + "name": "MRGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "MRGT", + "rawType": "float64", + "type": "float" + }, + { + "name": "MRGX", + "rawType": "float64", + "type": "float" + }, + { + "name": "REFR", + "rawType": "float64", + "type": "float" + }, + { + "name": "RMSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "RNTM", + "rawType": "float64", + "type": "float" + }, + { + "name": "RNTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "RWAT", + "rawType": "float64", + "type": "float" + }, + { + "name": "RWATPR", + "rawType": "float64", + "type": "float" + }, + { + "name": "SINK", + "rawType": "float64", + "type": "float" + }, + { + "name": "SMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "STOV", + "rawType": "float64", + "type": "float" + }, + { + "name": "TEL", + "rawType": "float64", + "type": "float" + }, + { + "name": "TEN", + "rawType": "float64", + "type": "float" + }, + { + "name": "TOIL", + "rawType": "float64", + "type": "float" + }, + { + "name": "VACS", + "rawType": "float64", + "type": "float" + }, + { + "name": "VALP", + "rawType": "float64", + "type": "float" + }, + { + "name": "VEH", + "rawType": "float64", + "type": "float" + }, + { + "name": "WATP", + "rawType": "float64", + "type": "float" + }, + { + "name": "YBL", + "rawType": "float64", + "type": "float" + }, + { + "name": "FES", + "rawType": "float64", + "type": "float" + }, + { + "name": "FINCP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FPARC", + "rawType": "float64", + "type": "float" + }, + { + "name": "GRNTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "GRPIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "HHL", + "rawType": "float64", + "type": "float" + }, + { + "name": "HUGCL", + "rawType": "float64", + "type": "float" + }, + { + "name": "HUPAC", + "rawType": "float64", + "type": "float" + }, + { + "name": "HUPAOC", + "rawType": "float64", + "type": "float" + }, + { + "name": "HUPARC", + "rawType": "float64", + "type": "float" + }, + { + "name": "KIT", + "rawType": "float64", + "type": "float" + }, + { + "name": "LNGI", + "rawType": "float64", + "type": "float" + }, + { + "name": "MULTG", + "rawType": "float64", + "type": "float" + }, + { + "name": "MV", + "rawType": "float64", + "type": "float" + }, + { + "name": "NOC", + "rawType": "float64", + "type": "float" + }, + { + "name": "NPF", + "rawType": "float64", + "type": "float" + }, + { + "name": "NPP", + "rawType": "float64", + "type": "float" + }, + { + "name": "NR", + "rawType": "float64", + "type": "float" + }, + { + "name": "NRC", + "rawType": "float64", + "type": "float" + }, + { + "name": "OCPIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "PARTNER", + "rawType": "float64", + "type": "float" + }, + { + "name": "PLM", + "rawType": "float64", + "type": "float" + }, + { + "name": "PSF", + "rawType": "float64", + "type": "float" + }, + { + "name": "R18", + "rawType": "float64", + "type": "float" + }, + { + "name": "R60", + "rawType": "float64", + "type": "float" + }, + { + "name": "R65", + "rawType": "float64", + "type": "float" + }, + { + "name": "RESMODE", + "rawType": "float64", + "type": "float" + }, + { + "name": "SMOCP", + "rawType": "float64", + "type": "float" + }, + { + "name": "SMX", + "rawType": "float64", + "type": "float" + }, + { + "name": "SRNT", + "rawType": "float64", + "type": "float" + }, + { + "name": "SVAL", + "rawType": "float64", + "type": "float" + }, + { + "name": "TAXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "WIF", + "rawType": "float64", + "type": "float" + }, + { + "name": "WKEXREL", + "rawType": "float64", + "type": "float" + }, + { + "name": "WORKSTAT", + "rawType": "float64", + "type": "float" + }, + { + "name": "FACRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FAGSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FBATHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FBDSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FBLDP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FBUSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FCONP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FELEP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FFSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FFULP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FGASP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FHFLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FINSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FKITP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMRGIP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMRGP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMRGTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMRGXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FMVP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FPLMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FREFRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRMSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRNTMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRNTP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRWATP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FRWATPRP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSINKP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSMP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSMXHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSMXSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FSTOVP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FTAXP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FTELP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FTENP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FTOILP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FVACSP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FVALP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FVEHP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FWATP", + "rawType": "float64", + "type": "float" + }, + { + "name": "FYBLP", + "rawType": "float64", + "type": "float" + }, + { + "name": "household_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "person_id", + "rawType": "int64", + "type": "integer" + }, + { + "name": "pemploy", + "rawType": "int64", + "type": "integer" + }, + { + "name": "GRADE", + "rawType": "float64", + "type": "float" + }, + { + "name": "pstudent", + "rawType": "int64", + "type": "integer" + }, + { + "name": "ptype", + "rawType": "int64", + "type": "integer" + } + ], + "ref": "4d876e7c-ae8c-4906-82a9-ef90fd3fc985", + "rows": [], + "shape": { + "columns": 366, + "rows": 0 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
bl10_idmuni_idmuni_namerpa_acrmpohidserialnoperson_numblock_idyear...FVALPFVEHPFWATPFYBLPhousehold_idperson_idpemployGRADEpstudentptype
\n", + "

0 rows × 366 columns

\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [bl10_id, muni_id, muni_name, rpa_acr, mpo, hid, serialno, person_num, block_id, year, age, is_worker, relationship_to_sp1, pinc2013, hhinc2013, inc_adj, wage_inc, hh_inc, persons, workers, tenure, child, income_grp, children, adult, adults, HHtype, ageCAT6, ageHHder, pid, AGEP.x, SPORDER.x, ageHH, ht, HHT, HHsize, children_in_HH, SERIALNO, SPORDER.y, AGEP.y, CIT, CITWP05, CITWP12, COW, DDRS, DEAR, DEYE, DOUT, DPHY, DRAT, DRATX, DREM, ENG, FER, GCL, GCM, GCR, HINS1, HINS2, HINS3, HINS4, HINS5, HINS6, HINS7, INTP, JWMNP, JWRIP, JWTR, LANX, MAR, MARHD, MARHM, MARHT, MARHW, MARHYP05, MARHYP12, MIG, MIL, MLPA, MLPB, MLPCD, MLPE, MLPFG, MLPH, MLPI, MLPJ, MLPK, NWAB, NWAV, NWLA, NWLK, NWRE, OIP, PAP, RELP, RETP, SCH, SCHG, SCHL, SEMP, ...]\n", + "Index: []\n", + "\n", + "[0 rows x 366 columns]" + ] + }, + "execution_count": 86, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# per[(per[\"age\"]<6)&(per[\"SCHG\"].isin([15,16]))]\n", + "per[per[\"ptype\"]==0]" + ] + }, + { + "cell_type": "code", + "execution_count": 87, "metadata": {}, "outputs": [ { @@ -1299,7 +3217,7 @@ "type": "integer" } ], - "ref": "6118d47a-ffcb-4b05-8aa8-8b56600f6c65", + "ref": "9ef1998e-d8bf-4950-8381-17b64ce0068f", "rows": [ [ "0", @@ -1460,7 +3378,7 @@ "4 5 4 2 13 2.0 4 1 7" ] }, - "execution_count": 119, + "execution_count": 87, "metadata": {}, "output_type": "execute_result" } @@ -1487,7 +3405,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 88, "metadata": {}, "outputs": [ { @@ -1499,12 +3417,12 @@ "ptype \n", "Full-time worker 2315798 34.13\n", "Part-time worker 686766 10.12\n", - "College student 453852 6.69\n", + "College student 452192 6.67\n", "Non-working adult 1060625 15.63\n", "Retired 884012 13.03\n", "Driving-age student 177565 2.62\n", "Non-driving student 762015 11.23\n", - "Child too young for school 443844 6.54\n", + "Child too young for school 445504 6.57\n", "\n", " count percent\n", "pemploy \n", @@ -1536,7 +3454,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 89, "metadata": {}, "outputs": [], "source": [ @@ -1545,7 +3463,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 90, "metadata": {}, "outputs": [], "source": [ @@ -1558,7 +3476,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 91, "metadata": {}, "outputs": [ { @@ -1567,7 +3485,7 @@ "6784477" ] }, - "execution_count": 123, + "execution_count": 91, "metadata": {}, "output_type": "execute_result" } @@ -1578,7 +3496,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 92, "metadata": {}, "outputs": [ { @@ -1587,7 +3505,7 @@ "6784477" ] }, - "execution_count": 124, + "execution_count": 92, "metadata": {}, "output_type": "execute_result" } @@ -1598,7 +3516,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 93, "metadata": {}, "outputs": [ { @@ -1607,7 +3525,7 @@ "np.int64(6784477)" ] }, - "execution_count": 125, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } @@ -1618,12 +3536,12 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 95, "metadata": {}, "outputs": [], "source": [ - "hh.to_csv(os.path.join(input_dir, \"processed\", \"households.csv\"), index=False)\n", - "per.to_csv(os.path.join(input_dir, \"processed\", \"persons.csv\"), index=False)" + "hh.to_csv(os.path.join(output_dir, \"households.csv\"), index=False)\n", + "per.to_csv(os.path.join(output_dir, \"persons.csv\"), index=False)" ] } ], From 9c39508aa691994c43d91caba476af8ba2081cec Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:07:03 -0700 Subject: [PATCH 07/10] make skim negative values to 0 --- notebooks/process_skims.ipynb | 58 ++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/notebooks/process_skims.ipynb b/notebooks/process_skims.ipynb index f039a17..5af2c83 100644 --- a/notebooks/process_skims.ipynb +++ b/notebooks/process_skims.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -16,23 +16,23 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "input_file_name = \"ta_md.omx\"\n", "output_file_name = \"ta_md_ev.omx\"\n", "\n", - "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\_skim\"\n", - "output_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\"\n", + "input_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\EXTERNAL\"\n", + "output_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\"\n", "\n", - "input_file = os.path.join(input_dir, input_file_name)\n", + "input_file = os.path.join(input_dir, \"_skim\", input_file_name)\n", "output_file = os.path.join(output_dir, output_file_name)" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -41,7 +41,7 @@ "['ID', 'RCIndex']" ] }, - "execution_count": 31, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -53,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -132,6 +132,7 @@ " matrix_out[pm_name] = data.T\n", " print(f\"Transposed and saved as {pm_name}\")\n", "\n", + "\n", "matrix_in.close()\n", "matrix_out.close()" ] @@ -145,7 +146,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -188,6 +189,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " md_name = f\"{matrix_name}__MD\"\n", " matrix_out[md_name] = data\n", " print(f\"Saved as {md_name}\")\n", @@ -197,6 +199,8 @@ " matrix_out[ev_name] = data.copy()\n", " print(f\"Copied and saved as {ev_name}\")\n", "\n", + "matrix_out[\"dist\"] = matrix_out[\"dist__MD\"][:]\n", + "\n", "matrix_in.close()\n", "matrix_out.close()" ] @@ -210,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -232,6 +236,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " new_name = \"dist_nm\"\n", " matrix_out[new_name] = data\n", " print(f\"Saved as {new_name}\")\n", @@ -249,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -300,6 +305,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " matrix_name = matrix_name.lower()\n", "\n", " am_name = f\"tw_{matrix_name}__AM\"\n", @@ -324,7 +330,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -375,6 +381,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " matrix_name = matrix_name.lower()\n", "\n", " md_name = f\"tw_{matrix_name}__MD\"\n", @@ -398,7 +405,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -461,6 +468,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " matrix_name = matrix_name.lower()\n", "\n", " am_name = f\"ta_{matrix_name}__AM\"\n", @@ -485,7 +493,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -548,6 +556,7 @@ " print(f\"process {matrix_name}\")\n", " \n", " data = matrix_in[matrix_name][:]\n", + " data[data < 0] = 0\n", " matrix_name = matrix_name.lower()\n", "\n", " md_name = f\"ta_{matrix_name}__MD\"\n", @@ -561,6 +570,27 @@ "matrix_in.close()\n", "matrix_out.close()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 81581c30ac7b9ab829e4c6694355b138e8aaf050 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Thu, 9 Jul 2026 15:27:20 -0700 Subject: [PATCH 08/10] update subarea inputs --- notebooks/prepare_subrea_inputs.ipynb | 74 ++++++++++++++------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/notebooks/prepare_subrea_inputs.ipynb b/notebooks/prepare_subrea_inputs.ipynb index c14a3ee..38e7df8 100644 --- a/notebooks/prepare_subrea_inputs.ipynb +++ b/notebooks/prepare_subrea_inputs.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -20,13 +20,14 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "landuse_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", - "population_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\synthetic_population\"\n", - "skim_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\"" + "project_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\"\n", + "\n", + "input_dir = os.path.join(project_dir, \"EXTERNAL\")\n", + "subarea_dir = os.path.join(project_dir, \"subarea_downtown\")" ] }, { @@ -38,17 +39,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "zone_file = gpd.read_file(os.path.join(landuse_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\"))\n", - "landuse_file = pd.read_csv(os.path.join(landuse_dir, \"processed\", \"land_use.csv\"))" + "zone_file = gpd.read_file(os.path.join(input_dir, \"zonal\", \"shp\", \"CTPS_TDM23_TAZ_2017g_v202303.shp\"))\n", + "landuse_file = pd.read_csv(os.path.join(project_dir, \"landuse\", \"land_use.csv\"))" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -70,7 +71,7 @@ "metadata": {}, "outputs": [], "source": [ - "landuse_file.to_csv(os.path.join(landuse_dir, \"processed\", \"land_use_subarea.csv\"), index=False)" + "landuse_file.to_csv(os.path.join(subarea_dir, \"landuse\", \"land_use_subarea.csv\"), index=False)" ] }, { @@ -86,13 +87,13 @@ "metadata": {}, "outputs": [], "source": [ - "hh_df = pd.read_csv(os.path.join(population_dir, \"processed\", \"households.csv\"))\n", - "pop_df = pd.read_csv(os.path.join(population_dir, \"processed\", \"persons.csv\"))" + "hh_df = pd.read_csv(os.path.join(project_dir, \"synthetic_population\", \"households.csv\"))\n", + "pop_df = pd.read_csv(os.path.join(project_dir, \"synthetic_population\", \"persons.csv\"))" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -101,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -110,12 +111,12 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "hh_df.to_csv(os.path.join(population_dir, \"processed\", \"households_subarea.csv\"), index=False)\n", - "pop_df.to_csv(os.path.join(population_dir, \"processed\", \"persons_subarea.csv\"), index=False)" + "hh_df.to_csv(os.path.join(subarea_dir, \"synthetic_population\", \"households_subarea.csv\"), index=False)\n", + "pop_df.to_csv(os.path.join(subarea_dir, \"synthetic_population\", \"persons_subarea.csv\"), index=False)" ] }, { @@ -127,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -146,14 +147,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_am_pm.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\hwy_am_pm.omx\n", "save matrix da_time__AM\n", "save matrix da_time__PM\n", "save matrix da_toll__AM\n", @@ -166,12 +167,13 @@ "save matrix sr_time__PM\n", "save matrix sr_toll__AM\n", "save matrix sr_toll__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\hwy_md_ev.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\hwy_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\hwy_md_ev.omx\n", "save matrix da_time__EV\n", "save matrix da_time__MD\n", "save matrix da_toll__EV\n", "save matrix da_toll__MD\n", + "save matrix dist\n", "save matrix dist__EV\n", "save matrix dist__MD\n", "save matrix rs_fare__EV\n", @@ -180,11 +182,11 @@ "save matrix sr_time__MD\n", "save matrix sr_toll__EV\n", "save matrix sr_toll__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\hwy_md_ev_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\nm_daily.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\hwy_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\nm_daily.omx\n", "save matrix dist_nm\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\nm_daily_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_am_pm.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\nm_daily_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\tw_am_pm.omx\n", "save matrix tw_fare__AM\n", "save matrix tw_fare__PM\n", "save matrix tw_gen_cost__AM\n", @@ -201,8 +203,8 @@ "save matrix tw_xfer__PM\n", "save matrix tw_xwait__AM\n", "save matrix tw_xwait__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\tw_md_ev.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\tw_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\tw_md_ev.omx\n", "save matrix tw_fare__EV\n", "save matrix tw_fare__MD\n", "save matrix tw_gen_cost__EV\n", @@ -219,8 +221,8 @@ "save matrix tw_xfer__MD\n", "save matrix tw_xwait__EV\n", "save matrix tw_xwait__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\tw_md_ev_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_am_pm.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\tw_md_ev_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\ta_am_pm.omx\n", "save matrix ta_auto_cost__AM\n", "save matrix ta_auto_cost__PM\n", "save matrix ta_ddist__AM\n", @@ -243,8 +245,8 @@ "save matrix ta_xfer__PM\n", "save matrix ta_xwait__AM\n", "save matrix ta_xwait__PM\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_am_pm_subarea.omx\n", - "process C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\ta_md_ev.omx\n", + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\ta_am_pm_subarea.omx\n", + "process C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\skims\\ta_md_ev.omx\n", "save matrix ta_auto_cost__EV\n", "save matrix ta_auto_cost__MD\n", "save matrix ta_ddist__EV\n", @@ -267,14 +269,14 @@ "save matrix ta_xfer__MD\n", "save matrix ta_xwait__EV\n", "save matrix ta_xwait__MD\n", - "create C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\skim_prep\\processed\\subarea\\ta_md_ev_subarea.omx\n" + "create C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\subarea_downtown\\skims\\ta_md_ev_subarea.omx\n" ] } ], "source": [ "for skim in skims:\n", - " input_omx = os.path.join(skim_dir, \"processed\", f\"{skim}.omx\")\n", - " output_omx = os.path.join(skim_dir, \"processed\", \"subarea\", f\"{skim}_subarea.omx\")\n", + " input_omx = os.path.join(project_dir, \"skims\", f\"{skim}.omx\")\n", + " output_omx = os.path.join(subarea_dir, \"skims\", f\"{skim}_subarea.omx\")\n", "\n", " print(f\"process {input_omx}\")\n", "\n", From 1ea76247f94eec7d039b031d6cbcd5f3933894bd Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:07:42 -0700 Subject: [PATCH 09/10] update landuse assumptions --- notebooks/process_landuse.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/notebooks/process_landuse.ipynb b/notebooks/process_landuse.ipynb index 1e28510..cc14412 100644 --- a/notebooks/process_landuse.ipynb +++ b/notebooks/process_landuse.ipynb @@ -114,14 +114,14 @@ "source": [ "Assumptions\n", "- COUNTY: county FIPS, externals use 99\n", - "- DISTRICT: district from shp\n", - "- SD: state from shp\n", - "- TOTACRE: total_area from shp, convert to acre\n", + "- DISTRICT: district from zone shp\n", + "- SD: state from zone shp\n", + "- TOTACRE: total_area from zone shp, convert to acre\n", "- RESACRE: TOTACRE*0.5\n", "- CIACRE: TOTACRE*0.1\n", - "- area_type: urban from shp\n", + "- area_type: transit access_density from access_density.csv\n", "- TOPOLOGY: 1 for all\n", - "- TERMINAL: 4 for urban, 1 or others" + "- TERMINAL: terminal_time_p from access_density.csv" ] }, { From ace1016ddf0bd926f983fb961f016aedc87f60d9 Mon Sep 17 00:00:00 2001 From: Yue Shuai <48269801+yueshuaing@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:08:00 -0700 Subject: [PATCH 10/10] remove MA HH and emp from NHRI dataset --- notebooks/process_landuse.ipynb | 1060 +++---------------------------- 1 file changed, 91 insertions(+), 969 deletions(-) diff --git a/notebooks/process_landuse.ipynb b/notebooks/process_landuse.ipynb index cc14412..f79f184 100644 --- a/notebooks/process_landuse.ipynb +++ b/notebooks/process_landuse.ipynb @@ -19,8 +19,8 @@ "metadata": {}, "outputs": [], "source": [ - "input_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\"\n", - "output_dir = r\"C:\\Users\\USYS671257\\OneDrive - WSP O365\\42_Boston_CTPS\\land_use_prep\\processed\"" + "input_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\EXTERNAL\"\n", + "output_dir = r\"C:\\Users\\USYS671257\\WSP O365\\Boston MPO Model Support - General\\7. Next Generation Model Advancement\\ActivitySim_Inputs\\landuse\"" ] }, { @@ -319,7 +319,7 @@ "type": "integer" } ], - "ref": "c30e7556-7b32-4a1f-b4c4-c3936ce9f710", + "ref": "91a9671a-5642-41c9-8d06-c4728ca70970", "rows": [ [ "0", @@ -1243,975 +1243,80 @@ "outputs": [], "source": [ "ma_emp_agg = process_employment_data(ma_emp_df, block_to_taz_df)\n", - "nhri_emp_agg = process_employment_data(nhri_emp_df, block_to_taz_df)\n", - "\n", - "emp_combined = pd.concat([ma_emp_agg, nhri_emp_agg], ignore_index=False)\n", - "emp_combined = emp_combined.set_index('TAZ')\n", - "emp_combined = emp_combined.groupby(level=0).sum().reset_index()\n", - "emp_combined = emp_combined.sort_values('TAZ').reset_index(drop=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "zone_shape_df = zone_shape_df.merge(emp_combined, on='TAZ', how='left').fillna(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## population" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "hid", - "rawType": "object", - "type": "string" - }, - { - "name": "block_id", - "rawType": "int64", - "type": "integer" - }, - { - "name": "hh_inc", - "rawType": "int64", - "type": "integer" - }, - { - "name": "persons", - "rawType": "int64", - "type": "integer" - }, - { - "name": "workers", - "rawType": "int64", - "type": "integer" - }, - { - "name": "children", - "rawType": "int64", - "type": "integer" - }, - { - "name": "person_num", - "rawType": "int64", - "type": "integer" - }, - { - "name": "age", - "rawType": "int64", - "type": "integer" - }, - { - "name": "is_worker", - "rawType": "int64", - "type": "integer" - }, - { - "name": "wage_inc", - "rawType": "int64", - "type": "integer" - } - ], - "ref": "a9bb9b5e-3f82-4135-8134-4b8c5724edb1", - "rows": [ - [ - "0", - "2009000000007_1", - "250214175021007", - "36590", - "1", - "1", - "0", - "1", - "43", - "1", - "36590" - ], - [ - "1", - "2009000000007_2", - "250214179021003", - "36590", - "1", - "1", - "0", - "1", - "43", - "1", - "36590" - ], - [ - "2", - "2009000000007_3", - "250214180031005", - "36590", - "1", - "1", - "0", - "1", - "43", - "1", - "36590" - ], - [ - "3", - "2009000000064_10", - "250214123004013", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "4", - "2009000000064_10", - "250214123004013", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "5", - "2009000000064_10", - "250214123004013", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "6", - "2009000000064_11", - "250214131001053", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "7", - "2009000000064_11", - "250214131001053", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "8", - "2009000000064_12", - "250214131001055", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "9", - "2009000000064_12", - "250214131001055", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "10", - "2009000000064_13", - "250214131004012", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "11", - "2009000000064_13", - "250214131004012", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "12", - "2009000000064_13", - "250214131004012", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "13", - "2009000000064_14", - "250214131004012", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "14", - "2009000000064_14", - "250214131004012", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "15", - "2009000000064_15", - "250214131005019", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "16", - "2009000000064_15", - "250214131005019", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "17", - "2009000000064_15", - "250214131005019", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "18", - "2009000000064_16", - "250214132002001", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "19", - "2009000000064_16", - "250214132002001", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "20", - "2009000000064_16", - "250214132002001", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "21", - "2009000000064_17", - "250214132004013", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "22", - "2009000000064_17", - "250214132004013", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "23", - "2009000000064_18", - "250214133003017", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "24", - "2009000000064_18", - "250214133003017", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "25", - "2009000000064_18", - "250214133003017", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "26", - "2009000000064_19", - "250214133003020", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "27", - "2009000000064_19", - "250214133003020", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "28", - "2009000000064_4", - "250173507006005", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "29", - "2009000000064_4", - "250173507006005", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "30", - "2009000000064_5", - "250173732001012", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "31", - "2009000000064_5", - "250173732001012", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "32", - "2009000000064_6", - "250173861002039", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "33", - "2009000000064_6", - "250173861002039", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "34", - "2009000000064_6", - "250173861002039", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "35", - "2009000000064_7", - "250214021014019", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "36", - "2009000000064_7", - "250214021014019", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "37", - "2009000000064_7", - "250214021014019", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "38", - "2009000000064_8", - "250214112002000", - "61456", - "2", - "1", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "39", - "2009000000064_8", - "250214112002000", - "61456", - "2", - "1", - "1", - "2", - "14", - "0", - "0" - ], - [ - "40", - "2009000000064_9", - "250214123002013", - "120076", - "3", - "2", - "1", - "1", - "42", - "1", - "61456" - ], - [ - "41", - "2009000000064_9", - "250214123002013", - "120076", - "3", - "2", - "1", - "2", - "44", - "1", - "58620" - ], - [ - "42", - "2009000000064_9", - "250214123002013", - "120076", - "3", - "2", - "1", - "3", - "14", - "0", - "0" - ], - [ - "43", - "2009000000393_20", - "250092021024011", - "61456", - "3", - "1", - "1", - "1", - "49", - "1", - "61456" - ], - [ - "44", - "2009000000393_20", - "250092021024011", - "61456", - "3", - "1", - "1", - "2", - "20", - "0", - "0" - ], - [ - "45", - "2009000000393_20", - "250092021024011", - "61456", - "3", - "1", - "1", - "3", - "15", - "0", - "0" - ], - [ - "46", - "2009000000393_21", - "250092022001007", - "61456", - "3", - "1", - "1", - "1", - "49", - "1", - "61456" - ], - [ - "47", - "2009000000393_21", - "250092022001007", - "61456", - "3", - "1", - "1", - "2", - "20", - "0", - "0" - ], - [ - "48", - "2009000000393_21", - "250092022001007", - "61456", - "3", - "1", - "1", - "3", - "15", - "0", - "0" - ], - [ - "49", - "2009000000393_22", - "250092022002034", - "61456", - "3", - "1", - "1", - "1", - "49", - "1", - "61456" - ] - ], - "shape": { - "columns": 10, - "rows": 6784306 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
hidblock_idhh_incpersonsworkerschildrenperson_numageis_workerwage_inc
02009000000007_125021417502100736590110143136590
12009000000007_225021417902100336590110143136590
22009000000007_325021418003100536590110143136590
32009000000064_10250214123004013120076321142161456
42009000000064_10250214123004013120076321244158620
.................................
67843012013001492699_2756389250092683004009101828330164111346
67843022013001492699_2756389250092683004009101828330264175638
67843032013001492699_2756389250092683004009101828330329114844
67843042013001492699_275639025009268300400926190220164111346
67843052013001492699_275639025009268300400926190220229114844
\n", - "

6784306 rows × 10 columns

\n", - "
" - ], + "nhri_emp_agg = process_employment_data(nhri_emp_df, block_to_taz_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [], + "source": [ + "ma_taz_list = zone_shape_df[zone_shape_df[\"SD\"].isin([25])].TAZ.tolist()\n", + "nhri_taz_list = zone_shape_df[zone_shape_df[\"SD\"].isin([33,44])].TAZ.tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": {}, + "outputs": [], + "source": [ + "ma_emp_agg = ma_emp_agg[ma_emp_agg[\"TAZ\"].isin(ma_taz_list)]\n", + "nhri_emp_agg = nhri_emp_agg[nhri_emp_agg[\"TAZ\"].isin(nhri_taz_list)]" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "emp_combined = pd.concat([ma_emp_agg, nhri_emp_agg], ignore_index=False)\n", + "emp_combined = emp_combined.set_index('TAZ')\n", + "emp_combined = emp_combined.groupby(level=0).sum().reset_index()\n", + "emp_combined = emp_combined.sort_values('TAZ').reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [], + "source": [ + "zone_shape_df = zone_shape_df.merge(emp_combined, on='TAZ', how='left').fillna(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { "text/plain": [ - " hid block_id hh_inc persons workers \\\n", - "0 2009000000007_1 250214175021007 36590 1 1 \n", - "1 2009000000007_2 250214179021003 36590 1 1 \n", - "2 2009000000007_3 250214180031005 36590 1 1 \n", - "3 2009000000064_10 250214123004013 120076 3 2 \n", - "4 2009000000064_10 250214123004013 120076 3 2 \n", - "... ... ... ... ... ... \n", - "6784301 2013001492699_2756389 250092683004009 101828 3 3 \n", - "6784302 2013001492699_2756389 250092683004009 101828 3 3 \n", - "6784303 2013001492699_2756389 250092683004009 101828 3 3 \n", - "6784304 2013001492699_2756390 250092683004009 26190 2 2 \n", - "6784305 2013001492699_2756390 250092683004009 26190 2 2 \n", - "\n", - " children person_num age is_worker wage_inc \n", - "0 0 1 43 1 36590 \n", - "1 0 1 43 1 36590 \n", - "2 0 1 43 1 36590 \n", - "3 1 1 42 1 61456 \n", - "4 1 2 44 1 58620 \n", - "... ... ... ... ... ... \n", - "6784301 0 1 64 1 11346 \n", - "6784302 0 2 64 1 75638 \n", - "6784303 0 3 29 1 14844 \n", - "6784304 0 1 64 1 11346 \n", - "6784305 0 2 29 1 14844 \n", - "\n", - "[6784306 rows x 10 columns]" + "np.float64(4483716.0)" ] }, - "execution_count": 17, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "ma_pop_df" + "zone_shape_df.TOTEMP.sum()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## population" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 53, "metadata": {}, "outputs": [], "source": [ @@ -2235,13 +1340,30 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ "ma_pop_agg = process_population_data(ma_pop_df, block_to_taz_df)\n", - "nhri_pop_agg = process_population_data(nhri_pop_df, block_to_taz_df)\n", - "\n", + "nhri_pop_agg = process_population_data(nhri_pop_df, block_to_taz_df)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [], + "source": [ + "ma_pop_agg = ma_pop_agg[ma_pop_agg[\"TAZ\"].isin(ma_taz_list)]\n", + "nhri_pop_agg = nhri_pop_agg[nhri_pop_agg[\"TAZ\"].isin(nhri_taz_list)]" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ "pop_combined = pd.concat([ma_pop_agg, nhri_pop_agg], ignore_index=False)\n", "pop_combined = pop_combined.set_index('TAZ')\n", "pop_combined = pop_combined.groupby(level=0).sum().reset_index()\n", @@ -2250,7 +1372,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ @@ -2266,7 +1388,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -2278,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 59, "metadata": {}, "outputs": [], "source": [ @@ -2296,7 +1418,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -2309,7 +1431,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -2328,7 +1450,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 63, "metadata": {}, "outputs": [], "source": [