VeniVidiVici
/
invest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feature/urban-nature-access' of https://github.com/natcap/invest into bugfix/1179-rename-greenspace-to-urbannature 

Conflicts:
	src/natcap/invest/urban_nature_access.py
	tests/test_urban_nature_access.py
This commit is contained in:
James Douglass 2023-02-15 14:45:16 -08:00
parent 8f53ca25ae d7a0e4846d
commit 090bd12eb1
6 changed files with 134 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.github/actions/setup_env/action.yml vendored
View File

@ -33,12 +33,6 @@ runs:
with:
python-version: ${{ inputs.python-version }}
- name: Setup conda environment
uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
channels: conda-forge
# save week number to use in next step
# save CONDA_PREFIX to GITHUB_ENV so it's accessible outside of shell commands
- name: Set environment variables
@ -65,19 +59,15 @@ runs:
echo "Will update environment using this environment.yml:"
cat environment.yml
# NOTE the post step that saves the cache will only run if the job succeeds
- name: Restore conda environment cache
id: condacache
uses: actions/cache@v2
- name: Setup conda environment
uses: mamba-org/provision-with-micromamba@main
with:
path: ${{ env.CONDA_PREFIX }}
key: ${{ runner.os }}${{ runner.arch }}-${{ env.WEEK }}-${{ hashFiles('environment.yml') }}
- name: Update environment with dependencies
if: steps.condacache.outputs.cache-hit != 'true'
shell: bash -l {0} # conda only available in login shell
run: conda env update --file environment.yml
environment-file: environment.yml
environment-name: env
channels: conda-forge
cache-env: true
cache-env-key: ${{ runner.os }}${{ runner.arch }}-${{ env.WEEK }}-${{ hashFiles('environment.yml') }}
- name: List conda environment
shell: bash -l {0} # conda only available in login shell
run: conda list
shell: bash -l {0}
run: micromamba list

4
.github/workflows/build-and-test.yml vendored
View File

@ -93,7 +93,7 @@ jobs:
- name: Compare conda environments
continue-on-error: true
run: |
conda list --export > conda-env.txt
micromamba list > conda-env.txt
diff ./conda-env.txt ./conda-env-artifact/conda-env.txt
- name: Build and install wheel
@ -376,7 +376,7 @@ jobs:
run: make install
- name: Build binaries
run: make ${{ matrix.binary-make-command }}
run: make CONDA=micromamba ${{ matrix.binary-make-command }}
- name: Install Node.js
uses: actions/setup-node@v2

4
Makefile
View File

@ -10,7 +10,7 @@ GIT_TEST_DATA_REPO_REV := f5e651c9ba0a012dc033b9c1d12d51e42f6f87b0
GIT_UG_REPO := https://github.com/natcap/invest.users-guide
GIT_UG_REPO_PATH := doc/users-guide
GIT_UG_REPO_REV := 3945cfb44b272358b45acd3a1f625f45bc609d64
GIT_UG_REPO_REV := 6b5fffaf16b63e01b01fdea11c2774f0f8cbda96
ENV = "./env"
ifeq ($(OS),Windows_NT)
@ -267,7 +267,7 @@ $(INVEST_BINARIES_DIR): | $(DIST_DIR) $(BUILD_DIR)
-$(RMDIR) $(BUILD_DIR)/pyi-build
-$(RMDIR) $(INVEST_BINARIES_DIR)
$(PYTHON) -m PyInstaller --workpath $(BUILD_DIR)/pyi-build --clean --distpath $(DIST_DIR) exe/invest.spec
$(CONDA) list --export > $(INVEST_BINARIES_DIR)/package_versions.txt
$(CONDA) list > $(INVEST_BINARIES_DIR)/package_versions.txt
$(INVEST_BINARIES_DIR)/invest list
# Documentation.

115
src/natcap/invest/urban_nature_access.py
View File

@ -10,6 +10,7 @@ import tempfile
import numpy
import numpy.testing
import pygeoprocessing
import pygeoprocessing.symbolic
import shapely.ops
import shapely.wkb
import taskgraph
@ -295,7 +296,9 @@ ARGS_SPEC = {
_OUTPUT_BASE_FILES = {
'urban_nature_supply': 'urban_nature_supply.tif',
'admin_boundaries': 'admin_boundaries.gpkg',
'urban_nature_balance': 'urban_nature_balance.tif',
'urban_nature_balance_percapita': 'urban_nature_balance_percapita.tif',
'urban_nature_balance_totalpop': 'urban_nature_balance_totalpop.tif',
'urban_nature_demand': 'urban_nature_demand.tif',
}
_INTERMEDIATE_BASE_FILES = {
@ -307,7 +310,6 @@ _INTERMEDIATE_BASE_FILES = {
'urban_nature_area': 'urban_nature_area.tif',
'urban_nature_population_ratio': 'urban_nature_population_ratio.tif',
'convolved_population': 'convolved_population.tif',
'urban_nature_supply_demand_budget': 'urban_nature_supply_demand_budget.tif',
'undersupplied_population': 'undersupplied_population.tif',
'oversupplied_population': 'oversupplied_population.tif',
'reprojected_admin_boundaries': 'reprojected_admin_boundaries.gpkg',
@ -518,6 +520,23 @@ def execute(args):
dependent_task_list=[]
)
# This _could_ be a raster_calculator operation, but the math is so simple
# that it seems like this could suffice.
_ = graph.add_task(
pygeoprocessing.symbolic.evaluate_raster_calculator_expression,
kwargs={
'expression': f"population * {float(args['urban_nature_demand'])}",
'symbol_to_path_band_map': {
'population': (file_registry['masked_population'], 1),
},
'target_nodata': FLOAT32_NODATA,
'target_raster_path': file_registry['urban_nature_demand'],
},
task_name='Calculate urban nature demand',
target_path_list=[file_registry['urban_nature_demand']],
dependent_task_list=[population_mask_task]
)
# If we're doing anything with population groups, rasterize the AOIs and
# create the proportional population rasters.
proportional_population_paths = {}
@ -885,8 +904,8 @@ def execute(args):
index=False, name=None))
urban_nature_supply_by_group_paths = {}
urban_nature_supply_by_group_tasks = []
urban_nature_supply_demand_by_group_paths = {}
urban_nature_supply_demand_by_group_tasks = []
urban_nature_balance_totalpop_by_group_paths = {}
urban_nature_balance_totalpop_by_group_tasks = []
supply_population_paths = {'over': {}, 'under': {}}
supply_population_tasks = {'over': {}, 'under': {}}
for pop_group, proportional_pop_path in (
@ -917,7 +936,7 @@ def execute(args):
# Calculate SUP_DEMi_cap for each population group.
per_cap_urban_nature_balance_pop_group_path = os.path.join(
output_dir,
f'urban_nature_balance_{pop_group}{suffix}.tif')
f'urban_nature_balance_percapita_{pop_group}{suffix}.tif')
per_cap_urban_nature_balance_pop_group_task = graph.add_task(
pygeoprocessing.raster_calculator,
kwargs={
@ -925,7 +944,7 @@ def execute(args):
(urban_nature_supply_to_group_path, 1),
(float(args['urban_nature_demand']), 'raw')
],
'local_op': _urban_nature_balance_op,
'local_op': _urban_nature_balance_percapita_op,
'target_raster_path':
per_cap_urban_nature_balance_pop_group_path,
'datatype_target': gdal.GDT_Float32,
@ -939,27 +958,27 @@ def execute(args):
urban_nature_supply_by_group_task,
])
urban_nature_supply_demand_by_group_path = os.path.join(
urban_nature_balance_totalpop_by_group_path = os.path.join(
intermediate_dir,
f'urban_nature_supply_demand_budget_{pop_group}{suffix}.tif')
urban_nature_supply_demand_by_group_paths[
pop_group] = urban_nature_supply_demand_by_group_path
urban_nature_supply_demand_by_group_tasks.append(graph.add_task(
f'urban_nature_balance_totalpop_{pop_group}{suffix}.tif')
urban_nature_balance_totalpop_by_group_paths[
pop_group] = urban_nature_balance_totalpop_by_group_path
urban_nature_balance_totalpop_by_group_tasks.append(graph.add_task(
pygeoprocessing.raster_calculator,
kwargs={
'base_raster_path_band_const_list': [
(per_cap_urban_nature_balance_pop_group_path, 1),
(proportional_pop_path, 1)
],
'local_op': _urban_nature_supply_demand_op,
'local_op': _urban_nature_balance_totalpop_op,
'target_raster_path': (
urban_nature_supply_demand_by_group_path),
urban_nature_balance_totalpop_by_group_path),
'datatype_target': gdal.GDT_Float32,
'nodata_target': FLOAT32_NODATA
},
task_name='Calculate per-capita urban nature supply-demand',
target_path_list=[
urban_nature_supply_demand_by_group_path],
urban_nature_balance_totalpop_by_group_path],
dependent_task_list=[
per_cap_urban_nature_balance_pop_group_task,
proportional_population_tasks[pop_group],
@ -1013,19 +1032,38 @@ def execute(args):
*pop_group_proportion_tasks.values(),
])
urban_nature_supply_demand_budget_task = graph.add_task(
per_capita_urban_nature_balance_task = graph.add_task(
pygeoprocessing.raster_calculator,
kwargs={
'base_raster_path_band_const_list': [
(file_registry['urban_nature_supply'], 1),
(float(args['urban_nature_demand']), 'raw')
],
'local_op': _urban_nature_balance_percapita_op,
'target_raster_path':
file_registry['urban_nature_balance_percapita'],
'datatype_target': gdal.GDT_Float32,
'nodata_target': FLOAT32_NODATA
},
task_name='Calculate per-capita urban nature balance',
target_path_list=[file_registry['urban_nature_balance_percapita']],
dependent_task_list=[
urban_nature_supply_task,
])
urban_nature_balance_totalpop_task = graph.add_task(
ndr._sum_rasters,
kwargs={
'raster_path_list':
list(urban_nature_supply_demand_by_group_paths.values()),
list(urban_nature_balance_totalpop_by_group_paths.values()),
'target_nodata': FLOAT32_NODATA,
'target_result_path':
file_registry['urban_nature_supply_demand_budget'],
file_registry['urban_nature_balance_totalpop'],
},
task_name='2SFCA - urban_nature supply-demand budget',
task_name='2SFCA - urban nature - total population',
target_path_list=[
file_registry['urban_nature_supply_demand_budget']],
dependent_task_list=urban_nature_supply_demand_by_group_tasks
file_registry['urban_nature_balance_totalpop']],
dependent_task_list=urban_nature_balance_totalpop_by_group_tasks
)
# Summary stats for RADIUS_OPT_POP_GROUP
@ -1035,7 +1073,7 @@ def execute(args):
'source_aoi_vector_path': file_registry['reprojected_admin_boundaries'],
'target_aoi_vector_path': file_registry['admin_boundaries'],
'urban_nature_sup_dem_paths_by_pop_group':
urban_nature_supply_demand_by_group_paths,
urban_nature_balance_totalpop_by_group_paths,
'proportional_pop_paths_by_pop_group':
proportional_population_paths,
'undersupply_by_pop_group': supply_population_paths['under'],
@ -1046,7 +1084,7 @@ def execute(args):
target_path_list=[file_registry['admin_boundaries']],
dependent_task_list=[
aoi_reprojection_task,
*urban_nature_supply_demand_by_group_tasks,
*urban_nature_balance_totalpop_by_group_tasks,
*proportional_population_tasks.values(),
*supply_population_tasks['under'].values(),
*supply_population_tasks['over'].values(),
@ -1064,34 +1102,35 @@ def execute(args):
(file_registry['urban_nature_supply'], 1),
(float(args['urban_nature_demand']), 'raw')
],
'local_op': _urban_nature_balance_op,
'target_raster_path': file_registry['urban_nature_balance'],
'local_op': _urban_nature_balance_percapita_op,
'target_raster_path':
file_registry['urban_nature_balance_percapita'],
'datatype_target': gdal.GDT_Float32,
'nodata_target': FLOAT32_NODATA
},
task_name='Calculate per-capita urban nature balance',
target_path_list=[file_registry['urban_nature_balance']],
target_path_list=[file_registry['urban_nature_balance_percapita']],
dependent_task_list=[
urban_nature_supply_task,
])
# This is "SUP_DEMi" from the user's guide
urban_nature_supply_demand_task = graph.add_task(
urban_nature_balance_totalpop_task = graph.add_task(
pygeoprocessing.raster_calculator,
kwargs={
'base_raster_path_band_const_list': [
(file_registry['urban_nature_balance'], 1),
(file_registry['urban_nature_balance_percapita'], 1),
(file_registry['masked_population'], 1)
],
'local_op': _urban_nature_supply_demand_op,
'local_op': _urban_nature_balance_totalpop_op,
'target_raster_path': (
file_registry['urban_nature_supply_demand_budget']),
file_registry['urban_nature_balance_totalpop']),
'datatype_target': gdal.GDT_Float32,
'nodata_target': FLOAT32_NODATA
},
task_name='Calculate per-capita urban nature supply-demand',
task_name='Calculate urban nature balance for the total population',
target_path_list=[
file_registry['urban_nature_supply_demand_budget']],
file_registry['urban_nature_balance_totalpop']],
dependent_task_list=[
per_capita_urban_nature_balance_task,
population_mask_task,
@ -1121,7 +1160,7 @@ def execute(args):
kwargs={
'base_raster_path_band_const_list': [
(proportional_pop_path, 1),
(file_registry['urban_nature_balance'], 1),
(file_registry['urban_nature_balance_percapita'], 1),
(op, 'raw'), # numpy element-wise comparator
],
'local_op': _filter_population,
@ -1132,7 +1171,7 @@ def execute(args):
task_name=f'Determine {supply_type}supplied populations',
target_path_list=[supply_population_path],
dependent_task_list=[
urban_nature_supply_demand_task,
per_capita_urban_nature_balance_task,
population_mask_task,
*list(proportional_population_tasks.values()),
]))
@ -1143,7 +1182,7 @@ def execute(args):
'source_aoi_vector_path': file_registry['reprojected_admin_boundaries'],
'target_aoi_vector_path': file_registry['admin_boundaries'],
'urban_nature_budget_path': file_registry[
'urban_nature_supply_demand_budget'], # TODO: is this the correct raster?
'urban_nature_balance_totalpop'],
'population_path': file_registry['masked_population'],
'undersupplied_populations_path': file_registry[
'undersupplied_population'],
@ -1157,7 +1196,7 @@ def execute(args):
dependent_task_list=[
population_mask_task,
aoi_reprojection_task,
urban_nature_supply_demand_task,
urban_nature_balance_totalpop_task,
*supply_population_tasks
])
@ -1688,7 +1727,7 @@ def _write_supply_demand_vector(source_aoi_vector_path, feature_attrs,
target_vector = None
def _urban_nature_balance_op(urban_nature_supply, urban_nature_demand):
def _urban_nature_balance_percapita_op(urban_nature_supply, urban_nature_demand):
"""Calculate the per-capita urban nature balance.
This is the amount of urban nature that each pixel has above (positive
@ -1712,8 +1751,8 @@ def _urban_nature_balance_op(urban_nature_supply, urban_nature_demand):
return balance
def _urban_nature_supply_demand_op(urban_nature_balance, population):
"""Calculate the supply/demand of urban nature per person.
def _urban_nature_balance_totalpop_op(urban_nature_balance, population):
"""Calculate the total population urban nature balance.
Args:
urban_nature_balance (numpy.array): The area of urban nature budgeted to

54
tests/test_recreation.py
View File

@ -266,60 +266,6 @@ class TestRecServer(unittest.TestCase):
aoi_path,
os.path.join(out_workspace_dir, 'test_aoi_for_subset.shp'))
@_timeout(30.0)
def test_empty_server(self):
"""Recreation test a client call to simple server."""
from natcap.invest.recreation import recmodel_server
from natcap.invest.recreation import recmodel_client
empty_point_data_path = os.path.join(
self.workspace_dir, 'empty_table.csv')
open(empty_point_data_path, 'w').close() # touch the file
# attempt to get an open port; could result in race condition but
# will be okay for a test. if this test ever fails because of port
# in use, that's probably why
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(('', 0))
port = sock.getsockname()[1]
sock.close()
sock = None
server_args = {
'hostname': 'localhost',
'port': port,
'raw_csv_point_data_path': empty_point_data_path,
'cache_workspace': self.workspace_dir,
'min_year': 2004,
'max_year': 2015,
}
server_thread = threading.Thread(
target=recmodel_server.execute, args=(server_args,))
server_thread.daemon = True
server_thread.start()
client_args = {
'aoi_path': os.path.join(
SAMPLE_DATA, 'test_aoi_for_subset.shp'),
'cell_size': 7000.0,
'hostname': 'localhost',
'port': port,
'compute_regression': False,
'start_year': '2005',
'end_year': '2014',
'grid_aoi': False,
'results_suffix': '',
'workspace_dir': self.workspace_dir,
}
recmodel_client.execute(client_args)
# testing for file existence seems reasonable since mostly we are
# testing that a local server starts and a client connects to it
_test_same_files(
os.path.join(REGRESSION_DATA, 'file_list_empty_local_server.txt'),
self.workspace_dir)
def test_local_aggregate_points(self):
"""Recreation test single threaded local AOI aggregate calculation."""
from natcap.invest.recreation import recmodel_server

49
tests/test_urban_nature_access.py
View File

@ -351,15 +351,16 @@ class UNATests(unittest.TestCase):
[50, 100],
[40.75, nodata]], dtype=numpy.float32)
urban_nature_budget = urban_nature_access._urban_nature_balance_op(
urban_nature_supply, urban_nature_demand)
urban_nature_budget = (
urban_nature_access._urban_nature_balance_percapita_op(
urban_nature_supply, urban_nature_demand))
expected_urban_nature_budget = numpy.array([
[nodata, 50.5],
[25, 50]], dtype=numpy.float32)
numpy.testing.assert_allclose(
urban_nature_budget, expected_urban_nature_budget)
supply_demand = urban_nature_access._urban_nature_supply_demand_op(
supply_demand = urban_nature_access._urban_nature_balance_totalpop_op(
urban_nature_budget, population)
expected_supply_demand = numpy.array([
[nodata, 100 * 50.5],
@ -565,7 +566,8 @@ class UNATests(unittest.TestCase):
urban_nature_access.execute(args)
summary_vector = gdal.OpenEx(
os.path.join(args['workspace_dir'], 'output', 'admin_boundaries.gpkg'))
os.path.join(args['workspace_dir'], 'output',
'admin_boundaries.gpkg'))
summary_layer = summary_vector.GetLayer()
self.assertEqual(summary_layer.GetFeatureCount(), 1)
summary_feature = summary_layer.GetFeature(1)
@ -637,7 +639,8 @@ class UNATests(unittest.TestCase):
urban_nature_access.execute(args)
summary_vector = gdal.OpenEx(
os.path.join(args['workspace_dir'], 'output', 'admin_boundaries.gpkg'))
os.path.join(args['workspace_dir'], 'output',
'admin_boundaries.gpkg'))
summary_layer = summary_vector.GetLayer()
self.assertEqual(summary_layer.GetFeatureCount(), 1)
summary_feature = summary_layer.GetFeature(1)
@ -731,6 +734,42 @@ class UNATests(unittest.TestCase):
for args in (uniform_args, split_urban_nature_args, pop_group_args):
urban_nature_access.execute(args)
# make sure the output dir contains the correct files.
for output_filename in (
urban_nature_access._OUTPUT_BASE_FILES.values()):
basename, ext = os.path.splitext(
os.path.basename(output_filename))
suffix = args['results_suffix']
filepath = os.path.join(args['workspace_dir'], 'output',
f'{basename}_{suffix}{ext}')
self.assertTrue(os.path.exists(filepath))
# check the urban_nature demand raster
population = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'intermediate',
f'masked_population_{suffix}.tif'))
demand = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output',
f'urban_nature_demand_{suffix}.tif'))
nodata = urban_nature_access.FLOAT32_NODATA
valid_pixels = ~utils.array_equals_nodata(population, nodata)
numpy.testing.assert_allclose(
(population[valid_pixels].sum() *
float(args['urban_nature_demand'])),
demand[valid_pixels].sum())
# check the total-population urban_nature balance
per_capita_balance = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output',
f'urban_nature_balance_percapita_{suffix}.tif'))
totalpop_balance = pygeoprocessing.raster_to_numpy_array(
os.path.join(args['workspace_dir'], 'output',
f'urban_nature_balance_totalpop_{suffix}.tif'))
numpy.testing.assert_allclose(
per_capita_balance[valid_pixels] * population[valid_pixels],
totalpop_balance[valid_pixels],
rtol=1e-5) # accommodate accumulation of numerical error
uniform_radius_supply = pygeoprocessing.raster_to_numpy_array(
os.path.join(uniform_args['workspace_dir'], 'output',
'urban_nature_supply_uniform.tif'))