VeniVidiVici
/
invest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Resolve merge conflicts

This commit is contained in:
Claire Simpson 2025-04-07 15:31:53 -06:00
parent dd95ce2cb8 84f6108b0a
commit 0d152f9409
104 changed files with 7950 additions and 5112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
.github/actions/setup_env/action.yml vendored
View File

@ -1,4 +1,4 @@
name: "Set up python and conda environment"
name: "Set up conda environment"
description:
"This action creates a conda environment containing a given python version
and set of requirements. It caches the environment and restores it from
@ -7,9 +7,11 @@ description:
requirements, and the number of the week in the year, so that the cache
refreshes weekly."
inputs:
python-version:
description: "Python version to install"
required: true
python:
description:
"Path to python executable to use to run convert-requirements-to-conda-yml.py"
required: false
default: 'python'
requirements-files:
description:
"List of requirements files to install from. May be separated by spaces
@ -25,14 +27,6 @@ inputs:
runs:
using: "composite"
steps:
# set up python that's used below to run convert-requirements-to-conda-yml.py
# after the environment is updated, 'python' will point to the
# conda-managed python and this python won't be used again
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: ${{ inputs.python-version }}
# save week number to use in next step
# save CONDA_PREFIX to GITHUB_ENV so it's accessible outside of shell commands
- name: Set environment variables
@ -45,7 +39,6 @@ runs:
shell: bash
run: |
# make sure each package is separated by a newline
echo "python=${{ inputs.python-version }}" > extra_requirements.txt
echo "${{ inputs.requirements }}" | xargs | tr " " "\n" >> extra_requirements.txt
- name: Combine all requirements into environment YML
@ -53,7 +46,7 @@ runs:
run: |
# make sure each file is separated by a space
REQUIREMENTS_FILES=$(echo "${{ inputs.requirements-files }}" | xargs | tr "\n" " ")
python ./scripts/convert-requirements-to-conda-yml.py \
${{ inputs.python }} ./scripts/convert-requirements-to-conda-yml.py \
extra_requirements.txt $REQUIREMENTS_FILES \
> environment.yml
echo "Will update environment using this environment.yml:"

92
.github/workflows/build-and-test.yml vendored
View File

@ -20,7 +20,7 @@ env:
# build: dependency of make install
# nomkl: make sure numpy w/out mkl
# setuptools_scm: needed for versioning to work
CONDA_DEFAULT_DEPENDENCIES: python-build nomkl setuptools_scm libsqlite<3.49.1 sqlite<3.49.1 # https://github.com/natcap/invest/issues/1797
CONDA_DEFAULT_DEPENDENCIES: python-build nomkl setuptools_scm 'libsqlite<3.49.1' 'sqlite<3.49.1' # https://github.com/natcap/invest/issues/1797
LATEST_SUPPORTED_PYTHON_VERSION: "3.13"
jobs:
@ -62,8 +62,8 @@ jobs:
- name: Lint with doc8
run: |
# Skip line-too-long errors (D001)
python -m doc8 --ignore D001 HISTORY.rst README_PYTHON.rst
# Skip line-too-long errors (D001)
python -m doc8 --ignore D001 HISTORY.rst README_PYTHON.rst
run-model-tests:
name: Run model tests
@ -73,7 +73,14 @@ jobs:
fail-fast: false
matrix:
python-version: [3.9, "3.10", "3.11", "3.12", "3.13"]
os: [windows-latest, macos-13]
os: [windows-latest, macos-13, ubuntu-latest]
include:
- os: ubuntu-latest
python-version: 3.9
platform-specific-dependencies: libxcrypt
- os: ubuntu-latest
python-version: "3.10"
platform-specific-dependencies: libxcrypt
steps:
- uses: actions/checkout@v4
with:
@ -92,9 +99,11 @@ jobs:
- name: Set up python environment
uses: ./.github/actions/setup_env
with:
python-version: ${{ matrix.python-version }}
requirements-files: requirements.txt requirements-dev.txt constraints_tests.txt
requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }}
requirements: |
${{ env.CONDA_DEFAULT_DEPENDENCIES }}
${{ matrix.platform-specific-dependencies }}
python=${{ matrix.python-version }}
- name: Download previous conda environment.yml
continue-on-error: true
@ -109,22 +118,40 @@ jobs:
path: ./conda-env-artifact
- name: Compare conda environments
if: matrix.os != 'ubuntu-latest'
run: |
micromamba list > conda-env.txt
# make sure that the exit code is always 0
# otherwise, an error appears in the action annotations
diff ./conda-env.txt ./conda-env-artifact/conda-env.txt || true
- name: Build and install wheel
- name: Build wheel
run: |
# uninstall InVEST if it was already in the restored cache
python -m pip uninstall -y natcap.invest
python -m build --wheel
NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" python -m build --wheel
ls -la dist
pip install $(find dist -name "natcap[._-]invest*.whl")
- name: Run model tests
run: make test
# This produces a wheel that should work on any distro with glibc>=2.39.
# This is a very recent version. If we want to support older versions, I
# suspect we would need to build GDAL from source on an appropriate
# system (such as a manylinux docker container) to ensure compatibility.
# Symbols used in libgdal are the cause of the high minimum version,
# possibly because of installing with conda.
- name: Audit and repair wheel for manylinux
if: matrix.os == 'ubuntu-latest'
run: |
ldd --version
pip install auditwheel
WHEEL=$(find dist -name "natcap[._-]invest*.whl")
auditwheel show $WHEEL
auditwheel repair $WHEEL --plat manylinux_2_39_x86_64 -w dist
rm $WHEEL # remove the original wheel
- name: Install wheel and run model tests
run: |
pip install $(find dist -name "natcap[._-]invest*.whl")
make test
- name: Upload wheel artifact
uses: actions/upload-artifact@v4
@ -158,9 +185,17 @@ jobs:
runs-on: ${{ matrix.os }}
needs: check-syntax-errors
strategy:
fail-fast: false
matrix:
python-version: [3.9, "3.10", "3.11", "3.12", "3.13"]
os: [windows-latest, macos-13]
os: [windows-latest, macos-13, ubuntu-latest]
include:
- os: ubuntu-latest
python-version: 3.9
platform-specific-dependencies: libxcrypt
- os: ubuntu-latest
python-version: "3.10"
platform-specific-dependencies: libxcrypt
steps:
- uses: actions/checkout@v4
with:
@ -168,9 +203,11 @@ jobs:
- uses: ./.github/actions/setup_env
with:
python-version: ${{ matrix.python-version }}
requirements-files: requirements.txt
requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} twine
requirements: |
${{ env.CONDA_DEFAULT_DEPENDENCIES }}
python=${{ matrix.python-version }}
twine ${{ matrix.platform-specific-dependencies }}
- name: Build source distribution
run: |
@ -180,12 +217,12 @@ jobs:
# .cpp files in addition to the .pyx files.
#
# Elevating any python warnings to errors to catch build issues ASAP.
python -W error -m build --sdist
NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" python -W error -m build --sdist
- name: Install from source distribution
run : |
# Install natcap.invest from the sdist in dist/
pip install $(find dist -name "natcap[._-]invest*")
NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" pip install $(find dist -name "natcap[._-]invest*")
# Model tests should cover model functionality, we just want
# to be sure that we can import `natcap.invest` here.
@ -233,12 +270,14 @@ jobs:
- uses: ./.github/actions/setup_env
with:
python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
requirements-files: requirements.txt
requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} pytest
requirements: |
${{ env.CONDA_DEFAULT_DEPENDENCIES }}
python=${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
pytest
- name: Make install
run: make install
run: NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" make install
- name: Validate sample data
run: make validate_sampledata
@ -265,12 +304,13 @@ jobs:
- name: Set up python environment
uses: ./.github/actions/setup_env
with:
python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
requirements-files: requirements.txt requirements-dev.txt
requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }}
requirements: |
${{ env.CONDA_DEFAULT_DEPENDENCIES }}
python=${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
- name: Make install
run: make install
run: NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" make install
- name: Set up node
uses: actions/setup-node@v3
@ -322,16 +362,18 @@ jobs:
- name: Set up conda environment
uses: ./.github/actions/setup_env
with:
python-version: ${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
requirements-files: |
requirements.txt
requirements-dev.txt
requirements-docs.txt
constraints_tests.txt
requirements: ${{ env.CONDA_DEFAULT_DEPENDENCIES }} pandoc
requirements: |
${{ env.CONDA_DEFAULT_DEPENDENCIES }}
python=${{ env.LATEST_SUPPORTED_PYTHON_VERSION }}
pandoc
- name: Make install
run: make install
run: NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library" make install
# Not caching chocolatey packages because the cache may not be reliable
# https://github.com/chocolatey/choco/issues/2134

267
HISTORY.rst
View File

@ -1,5 +1,5 @@
..
Changes should be grouped for readability.
Changes should be grouped under headings for readability.
InVEST model names:
- Annual Water Yield
@ -9,7 +9,7 @@
- Crop Pollination
- Crop Production
- DelineateIt
- Forest Carbon Edge Effects
- Forest Carbon Edge Effect
- Globio
- Habitat Quality
- HRA
@ -33,49 +33,230 @@
Everything else:
- General
.. :changelog:
Updates worth drawing extra attention to (minor/major releases only):
- Highlights
Each release section has a heading underlined with ``---``.
Within a release section, underline group headings with ``===``.
For example:
Unreleased Changes
------------------
General
=======
* Updated something.
Wind Energy
===========
* Fixed something.
The order of groups should be as follows:
1. Highlights
2. General
3. Workbench
4. InVEST model A
5. ...
6. InVEST model Z (model names should be sorted A-Z)
Unreleased Changes
------------------
* General
* Fixed an issue where a user's PROJ_DATA environment variable could
trigger a RuntimeError about a missing proj.db file.
https://github.com/natcap/invest/issues/1742
* Now testing and building against Python 3.13.
No longer testing and building with Python 3.8, which reached EOL.
https://github.com/natcap/invest/issues/1755
* InVEST's windows binaries are now distributed once again with a valid
signature, signed by Stanford University.
https://github.com/natcap/invest/issues/1580
* InVEST's mac disk image is now distributed once again with a valid
signature, signed by Stanford University.
https://github.com/natcap/invest/issues/1784
* Annual Water Yield
* Fixed an issue where the model would crash if the valuation table was
provided, but the demand table was not. Validation will now warn about
this, and the ``MODEL_SPEC`` has been improved to reflect that this table
is now required when doing valuation.
https://github.com/natcap/invest/issues/1769
* Carbon
* Updated styling of the HTML report generated by the carbon model, for
visual consistency with the Workbench (`InVEST #1732
<https://github.com/natcap/invest/issues/1732>`_).
* Urban Cooling
* Updated the documentation for the ``mean_t_air`` attribute of the
``buildings_with_stats.shp`` output to clarify how the value is
calculated. https://github.com/natcap/invest/issues/1746
* Fixed bug in the calculation of Cooling Capacity (CC) provided by parks,
where the CC Index was not being properly incorporated.
https://github.com/natcap/invest/issues/1726
* Urban Stormwater Retention
* Fixed a bug causing ``inf`` values in volume outputs because nodata
values were not being set correctly (`InVEST #1850
<https://github.com/natcap/invest/issues/1850>`_).
* Wind Energy
* Fixed a bug that could cause the Workbench to crash when running the Wind
Energy model with ``Taskgraph`` logging set to ``DEBUG`` (`InVEST #1497
<https://github.com/natcap/invest/issues/1497>`_).
..
Unreleased Changes
------------------
3.15.0 (2025-04-03)
-------------------
Highlights
==========
* Multiple models now use **per-hectare** units in their raster outputs. Prior
to this update, these rasters reported **per-pixel** values. This change
affects the following models:
* Carbon
* Crop Production
* Forest Carbon Edge Effect
* NDR
* SDR
* NDR, SDR, and Seasonal Water Yield now support the D8 routing algorithm
in addition to MFD.
* Visitation: Recreation and Tourism model now includes Twitter data.
* InVEST model outputs now include metadata. Open the '.yml' files
in a text editor to read and add to the metadata.
General
=======
* Fixed an issue where a user's PROJ_DATA environment variable could
trigger a RuntimeError about a missing proj.db file.
https://github.com/natcap/invest/issues/1742
* Now testing and building against Python 3.13.
No longer testing and building with Python 3.8, which reached EOL.
https://github.com/natcap/invest/issues/1755
* All InVEST model output data now include metadata sidecar files.
These are '.yml' files with the same basename as the dataset they
describe. https://github.com/natcap/invest/issues/1662
* InVEST's Windows binaries are now distributed once again with a valid
signature, signed by Stanford University.
https://github.com/natcap/invest/issues/1580
* InVEST's macOS disk image is now distributed once again with a valid
signature, signed by Stanford University.
https://github.com/natcap/invest/issues/1784
* The natcap.invest python package now officially supports linux.
manylinux wheels will be available on PyPI.
(`#1730 <https://github.com/natcap/invest/issues/1730>`_)
* Removed the warning about ``gdal.UseExceptions()``.
Python API users should still call ``gdal.UseExceptions()``, but no
longer need to do so before importing ``natcap.invest``.
https://github.com/natcap/invest/issues/1702
Workbench
=========
* Auto-scrolling of log output is halted on user-initiated scrolling,
enabling easier inspection of log output while a model is running
(`InVEST #1533 <https://github.com/natcap/invest/issues/1533>`_).
* Fixed a bug where toggle inputs would fail to respond if multiple tabs
of the same model were open.
https://github.com/natcap/invest/issues/1842
Annual Water Yield
==================
* Fixed an issue where the model would crash if the valuation table was
provided, but the demand table was not. Validation will now warn about
this, and the ``MODEL_SPEC`` has been improved to reflect that this table
is now required when doing valuation.
https://github.com/natcap/invest/issues/1769
Carbon
======
* Updated styling of the HTML report generated by the carbon model, for
visual consistency with the Workbench (`InVEST #1732
<https://github.com/natcap/invest/issues/1732>`_).
* Raster outputs that previously contained per-pixel values (e.g., t/pixel)
now contain per-hectare values (e.g., t/ha). (`InVEST #1270
<https://github.com/natcap/invest/issues/1270>`_).
* Removed the REDD scenario and updated the naming of the Current and
Future scenarios to Baseline and Alternate, respectively, to better
indicate that users are not limited to comparing present and future.
(`InVEST #1758 <https://github.com/natcap/invest/issues/1758>`_).
* Changed output filename prefixes from ``tot_c`` to ``c_storage`` and
``delta`` to ``c_change``. (`InVEST #1825
<https://github.com/natcap/invest/issues/1825>`_).
* Fixed bug where discount rate and annual price change were incorrectly
treated as ratios instead of percentages. (`InVEST #1827
<https://github.com/natcap/invest/issues/1827>`_).
Coastal Blue Carbon
===================
* The ``code`` column in the model's biophysical table input, as well as
the ``code`` column in the preprocessor's LULC lookup table input and
``carbon_pool_transient_template`` output, have been renamed ``lucode``,
for consistency with other InVEST models (`InVEST #1249
<https://github.com/natcap/invest/issues/1249>`_).
Crop Production
===============
* Raster outputs that previously contained per-pixel values (e.g., t/pixel)
now contain per-hectare values (e.g., t/ha). This change affects both
the Percentile and Regression models (`InVEST #1270
<https://github.com/natcap/invest/issues/1270>`_).
Forest Carbon Edge Effect
=========================
* Raster outputs that previously contained per-pixel values (e.g., t/pixel)
now contain per-hectare values (e.g., t/ha). (`InVEST #1270
<https://github.com/natcap/invest/issues/1270>`_).
Habitat Quality
===============
* The ``lulc`` column in the sensitivity table input, and the ``lulc_code``
column in the rarity table outputs, have been renamed ``lucode``, for
consistency with other InVEST models (`InVEST #1249
<https://github.com/natcap/invest/issues/1249>`_).
* The model now expects the maximum threat distance (``max_dist`` in the
threats table) to be specified in ``m`` instead of ``km`` (`InVEST #1252
<https://github.com/natcap/invest/issues/1252>`_).
* Adjusted total habitat degradation calculation to calculate degradation
for each threat and create intermediate degradation rasters. Total
degradation is now calculated using these individual threat degradation
rasters.
https://github.com/natcap/invest/issues/1100
NDR
===
* Align rasters to the grid of the DEM raster
(`#1488 <https://github.com/natcap/invest/issues/1488>`_).
* Raster outputs that previously contained per-pixel values (e.g., kg/pixel)
now contain per-hectare values (e.g., kg/ha). (`InVEST #1270
<https://github.com/natcap/invest/issues/1270>`_).
* Made the runoff proxy index calculation more robust by allowing users to
specify the average runoff proxy, preventing normalization issues across
different climate scenarios and watershed selections.
https://github.com/natcap/invest/issues/1741
* D8 routing is now supported in addition to MFD
(`#1440 <https://github.com/natcap/invest/issues/1440>`_).
Scenario Generator
==================
* Updated the output CSV columns: Renamed `lucode` column `original lucode`
to clarify that it contains the original, to-be-converted, value(s). Added
`replacement lucode` column, containing the LULC code to which habitat was
converted during the model run.
https://github.com/natcap/invest/issues/1295
Scenic Quality
==============
* Fixed a bug where the visibility raster could be incorrectly set to 1
('visible') if the DEM value was within floating point imprecision of the
DEM nodata value (`#1859 <https://github.com/natcap/invest/issues/1859>`_).
SDR
===
* Raster outputs that previously contained per-pixel values (e.g., t/pixel)
now contain per-hectare values (e.g., t/ha). (`InVEST #1270
<https://github.com/natcap/invest/issues/1270>`_).
* D8 routing is now supported in addition to MFD
(`#1440 <https://github.com/natcap/invest/issues/1440>`_).
Seasonal Water Yield
====================
* D8 routing is now supported in addition to MFD
(`#1440 <https://github.com/natcap/invest/issues/1440>`_).
Urban Cooling
=============
* Align rasters to the grid of the LULC raster, rather than the ET0 raster
(`#1488 <https://github.com/natcap/invest/issues/1488>`_).
* Updated the documentation for the ``mean_t_air`` attribute of the
``buildings_with_stats.shp`` output to clarify how the value is
calculated. https://github.com/natcap/invest/issues/1746
* Fixed bug in the calculation of Cooling Capacity (CC) provided by parks,
where the CC Index was not being properly incorporated.
https://github.com/natcap/invest/issues/1726
Urban Stormwater Retention
==========================
* Fixed a bug causing ``inf`` values in volume outputs because nodata
values were not being set correctly (`InVEST #1850
<https://github.com/natcap/invest/issues/1850>`_).
Visitation: Recreation and Tourism
==================================
* Added a database of geotagged tweets to support calculating
twitter-user-days (TUD) as proxy for visitation rates. The model now calculates
photo-user-days (PUD) and TUD and uses their average as the response
variable in the regression model. Please refer to the User's Guide for
more details on the regression model.
* Output data were updated to support the new TUD results, and vector outputs
are now in GeoPackage format instead of ESRI Shapefile.
* Regression coefficients are still listed in a summary text file, and are now
also included in a tabular output: "regression_coefficients.csv".
Wind Energy
===========
* Fixed a bug that could cause the Workbench to crash when running the Wind
Energy model with ``Taskgraph`` logging set to ``DEBUG`` (`InVEST #1497
<https://github.com/natcap/invest/issues/1497>`_).
3.14.3 (2024-12-19)
-------------------

6
Makefile
View File

@ -2,15 +2,15 @@
DATA_DIR := data
GIT_SAMPLE_DATA_REPO := https://bitbucket.org/natcap/invest-sample-data.git
GIT_SAMPLE_DATA_REPO_PATH := $(DATA_DIR)/invest-sample-data
GIT_SAMPLE_DATA_REPO_REV := 0f8b41557753dad3670ba8220f41650b51435a93
GIT_SAMPLE_DATA_REPO_REV := ecdab62bd6e2d3d9105e511cfd6884bf07f3d27b
GIT_TEST_DATA_REPO := https://bitbucket.org/natcap/invest-test-data.git
GIT_TEST_DATA_REPO_PATH := $(DATA_DIR)/invest-test-data
GIT_TEST_DATA_REPO_REV := 324abde73e1d770ad75921466ecafd1ec6297752
GIT_TEST_DATA_REPO_REV := f0ebe739207ae57ae53a285d0fd954d6e8cfee54
GIT_UG_REPO := https://github.com/natcap/invest.users-guide
GIT_UG_REPO_PATH := doc/users-guide
GIT_UG_REPO_REV := fd3194f35bc1652a93cf1c0241f98be1e7c9d43d
GIT_UG_REPO_REV := 7d83c5bf05f0bef8dd4d2a4bd2f565ecf270af75
ENV = "./env"
ifeq ($(OS),Windows_NT)

5
README.rst
View File

@ -105,6 +105,11 @@ Building InVEST Distributions
-----------------------------
Once the required tools and packages are available, we can build InVEST.
On Windows, you must indicate the location of the GDAL libraries with the environment
variable ``NATCAP_INVEST_GDAL_LIB_PATH``. If you are using conda to manage dependencies
as we recommend, you can add ``NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library"`` to
the commands below. (On Mac and Linux, the gdal library path is determined for you
automatically using ``gdal-config``, which isn't available on Windows.)
Building ``natcap.invest`` python package

4
docker/Dockerfile
View File

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1
# Build the InVEST wheel in a separate container stage
FROM debian:12.2 as build
FROM debian:12.2 AS build
ARG INVEST_VERSION="main"
ARG INVEST_REPO="natcap/invest"
RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv
RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv libgdal-dev
RUN cd / && \
git clone https://github.com/${INVEST_REPO}.git && \
cd $(basename ${INVEST_REPO}) && \

13
exe/invest.spec
View File

@ -13,9 +13,16 @@ exename = 'invest'
conda_env = os.environ['CONDA_PREFIX']
if is_win:
proj_datas = ((os.path.join(conda_env, 'Library/share/proj'), 'proj'))
proj_datas = (os.path.join(conda_env, 'Library/share/proj'), 'proj')
frictionless_datas = (
os.path.join(conda_env, 'Lib/site-packages/frictionless/assets'),
'frictionless/assets')
else:
proj_datas = ((os.path.join(conda_env, 'share/proj'), 'proj'))
proj_datas = (os.path.join(conda_env, 'share/proj'), 'proj')
frictionless_datas = (
glob.glob(os.path.join(
conda_env, 'lib/python3*/site-packages/frictionless/assets'))[0],
'frictionless/assets')
kwargs = {
'hookspath': [os.path.join(current_dir, 'exe', 'hooks')],
@ -34,7 +41,7 @@ kwargs = {
'scipy.special._special_ufuncs',
'scipy._lib.array_api_compat.numpy.fft',
],
'datas': [proj_datas],
'datas': [proj_datas, frictionless_datas],
'cipher': block_cipher,
}

4
pyproject.toml
View File

@ -3,7 +3,8 @@ name = "natcap.invest"
description = "InVEST Ecosystem Service models"
readme = "README_PYTHON.rst"
requires-python = ">=3.9"
license = {file = "LICENSE.txt"}
license = "Apache-2.0"
license-files = ["LICENSE.txt"]
maintainers = [
{name = "Natural Capital Project Software Team"}
]
@ -23,7 +24,6 @@ classifiers = [
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Cython",
"License :: OSI Approved :: BSD License",
"Topic :: Scientific/Engineering :: GIS"
]
# the version is provided dynamically by setuptools_scm

5
requirements.txt
View File

@ -11,13 +11,13 @@
# on pip.
GDAL>=3.4.2
Pyro4==4.77 # pip-only
Pyro5
pandas>=1.2.1
numpy>=1.11.0,!=1.16.0
Rtree>=0.8.2,!=0.9.1
shapely>=2.0.0
scipy>=1.9.0,!=1.12.*
pygeoprocessing>=2.4.6 # pip-only
pygeoprocessing>=2.4.6
taskgraph>=0.11.0
psutil>=5.6.6
chardet>=3.0.4
@ -26,3 +26,4 @@ Babel
Flask
flask_cors
requests
geometamaker

7
scripts/convert-requirements-to-conda-yml.py
View File

@ -94,8 +94,11 @@ def build_environment_from_requirements(cli_args):
conda_deps_string = '\n'.join(
[f'- {dep}' for dep in sorted(conda_requirements, key=str.casefold)])
pip_deps_string = '- pip:\n' + '\n'.join(
[' - %s' % dep for dep in sorted(pip_requirements, key=str.casefold)])
if pip_requirements:
pip_deps_string = '- pip:\n' + '\n'.join(
[' - %s' % dep for dep in sorted(pip_requirements, key=str.casefold)])
else:
pip_deps_string = ''
print(YML_TEMPLATE.format(
conda_dependencies=conda_deps_string,
pip_dependencies=pip_deps_string))

53
scripts/recreation_server/build_twitter_quadtree.py Normal file
View File

@ -0,0 +1,53 @@
import argparse
import logging
import os
import sys
import natcap.invest.utils
from natcap.invest.recreation import recmodel_server
LOGGER = logging.getLogger(__name__)
root_logger = logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
filehandler = logging.FileHandler('logfile.txt', 'w', encoding='UTF-8')
formatter = logging.Formatter(
fmt=natcap.invest.utils.LOG_FMT,
datefmt='%m/%d/%Y %H:%M:%S ')
handler.setFormatter(formatter)
filehandler.setFormatter(formatter)
logging.basicConfig(level=logging.INFO, handlers=[handler, filehandler])
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'--csv_file_list', type=str,
help='path to text file with list of csv filepaths containing point data')
parser.add_argument(
'-w', '--workspace', type=str,
help='path to directory for writing quadtree files.')
parser.add_argument(
'-o', '--output_filename', type=str,
help='name for the pickle file quadtree index created in the workspace.')
parser.add_argument(
'-c', '--n_cores', type=int,
help='number of available cores for multiprocessing')
args = parser.parse_args()
with open(args.csv_file_list, 'r') as file:
csv_list = [line.rstrip() for line in file]
ooc_qt_pickle_filename = os.path.join(
args.workspace, args.output_filename)
recmodel_server.construct_userday_quadtree(
recmodel_server.INITIAL_BOUNDING_BOX,
csv_list,
'twitter',
args.workspace,
ooc_qt_pickle_filename,
recmodel_server.GLOBAL_MAX_POINTS_PER_NODE,
recmodel_server.GLOBAL_DEPTH,
n_workers=args.n_cores,
build_shapefile=False,
fast_point_count=True)

35
scripts/recreation_server/build_twitter_quadtree.sh Normal file
View File

@ -0,0 +1,35 @@
#!/bin/bash
# ----------------SLURM Parameters----------------
#SBATCH --partition normal,hns
#SBATCH --ntasks 1
#SBATCH --cpus-per-task=32
#SBATCH --mem-per-cpu=8000M
#SBATCH --nodes 1
# Define how long the job will run d-hh:mm:ss
#SBATCH --time=48:00:00
# Get email notification when job finishes or fails
#SBATCH --mail-user=
#SBATCH --mail-type=END,FAIL,BEGIN
#SBATCH -J build_quadtree
#SBATCH -o build_quadtree
# ----------------Load Modules--------------------
# ----------------Commands------------------------
# If this script is re-used, the user should expect
# to update these values.
CONTAINER=ghcr.io/natcap/invest:3.15.0
TWEETS_DIR=/scratch/users/woodsp/invest/csv
TWEETS_LIST=$SCRATCH/tweets_full_list.txt
find $TWEETS_DIR -name '*.csv' > $TWEETS_LIST
# invest repo already cloned into ~/invest
cd ~/invest
git checkout exp/REC-twitter
set -x # Be eXplicit about what's happening.
singularity run \
docker://$CONTAINER python scripts/recreation_server/build_twitter_quadtree.py \
--csv_file_list=$TWEETS_LIST \
--workspace=$SCRATCH/twitter_quadtree \
--output_filename=global_twitter_qt.pickle \
--n_cores=32 # match --cpus-per-task value

15
scripts/recreation_server/copy_quadtree_to_gcs.sh Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
#
#SBATCH --time=2:00:00
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=1
#SBATCH --mem-per-cpu=4000M
#SBATCH --mail-type=ALL
#SBATCH --mail-user=
#SBATCH --partition=hns,normal
#SBATCH --output=slurm-%j.%x.out
ml load system rclone
rclone copy --progress \
$SCRATCH/twitter_quadtree/ \
gcs-remote:natcap-recreation/twitter_quadtree/

6
scripts/recreation_server/cron_find_rm_cached_workspaces.sh Normal file → Executable file
View File

@ -1,10 +1,10 @@
# Setting this up to run on cron:
# make sure this script is executeable
# sudo crontab -e
# Enter: @daily /usr/local/recreation-server/find_remove_cached_workspaces.sh
# Enter: @daily /path/to/this/cron/script
# The following finds workspace directories that have not been modified in the past 7 days, and removes them.
# Right now we have two cache dirs going, for two rec servers.
find /usr/local/recreation-server/recserver_cache_py36/ -maxdepth 1 -mindepth 1 -type d -mtime +7 -regextype posix-egrep -regex "\/usr\/local\/recreation-server\/recserver_cache_py36\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" -exec rm -r {} \;
find /usr/local/recreation-server/recserver_cache_2017/ -maxdepth 1 -mindepth 1 -type d -mtime +7 -regextype posix-egrep -regex "\/usr\/local\/recreation-server\/recserver_cache_2017\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" -exec rm -r {} \;
find /usr/local/recreation-server/invest_3_15_0/server/flickr/local/ -maxdepth 1 -mindepth 1 -type d -mtime +7 -regextype posix-egrep -regex "\/usr\/local\/recreation-server\/invest_3_15_0\/server\/flickr\/local\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" -exec rm -r {} \;
find /usr/local/recreation-server/invest_3_15_0/server/twitter/local/ -maxdepth 1 -mindepth 1 -type d -mtime +7 -regextype posix-egrep -regex "\/usr\/local\/recreation-server\/invest_3_15_0\/server\/twitter\/local\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" -exec rm -r {} \;

35
scripts/recreation_server/execute_recmodel_server.py
View File

@ -1,5 +1,7 @@
import argparse
import logging
import multiprocessing
import os
import natcap.invest.recreation.recmodel_server
@ -7,20 +9,37 @@ logging.basicConfig(
format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s',
level=logging.DEBUG, datefmt='%m/%d/%Y %H:%M:%S ')
args = {
'hostname': '', # the local IP for the server
'port': 54322,
'raw_csv_point_data_path': 'photos_2005-2017_odlla.csv',
'max_year': 2017,
'min_year': 2005,
'cache_workspace': './recmodel_server_cache' # a local directory
args_dict = {
'hostname': '',
'port': 54322, # http://data.naturalcapitalproject.org/server_registry/invest_recreation_model_twitter/
'max_allowable_query': 40_000_000,
'datasets': {
'flickr': {
'raw_csv_point_data_path': '/usr/local/recreation-server/invest_3_15_0/server/volume/flickr/photos_2005-2017_odlla.csv',
'min_year': 2005,
'max_year': 2017
},
'twitter': {
'quadtree_pickle_filename': '/usr/local/recreation-server/invest_3_15_0/server/volume/twitter_quadtree/global_twitter_qt.pickle',
'min_year': 2012,
'max_year': 2022
}
}
}
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'-w', dest='cache_workspace',
help='Path to a local, writeable, directory. Avoid mounted volumes.')
args = parser.parse_args()
# It's crucial to specify `spawn` here as some OS use 'fork' as the default
# for new child processes. And a 'fork' will immediately duplicate all
# the resources (memory) of the parent. We noticed that causing
# Cannot Allocate Memory errors, as the recmodel_server can start child
# processes at a point when it's already using a lot of memory.
multiprocessing.set_start_method('spawn')
natcap.invest.recreation.recmodel_server.execute(args)
args_dict['cache_workspace'] = args.cache_workspace
mounted_volume = os.path.join(args.cache_workspace, 'cache')
natcap.invest.recreation.recmodel_server.execute(args_dict)

9
scripts/recreation_server/launch_recserver.sh
View File

@ -1,4 +1,7 @@
#!/bin/bash
source /home/davemfish/miniconda3/etc/profile.d/conda.sh
conda activate ./invest/env/
nohup python -u execute_recmodel_server.py > nohup_recmodel_server.txt 2>&1 &
# activate a python environment before calling this script:
# mamba activate /usr/local/recreation-server/invest_3_15_0/invest/env
# I think I had a hard time doing the activation within this script, for some reason.
nohup python -u /usr/local/recreation-server/invest_3_15_0/invest/scripts/recreation_server/execute_recmodel_server.py \
-w /usr/local/recreation-server/invest_3_15_0/server > \
/usr/local/recreation-server/invest_3_15_0/server/log.txt 2>&1 &

65
scripts/recreation_server/readme.txt
View File

@ -1,57 +1,14 @@
To start the rec server: sudo ./launch_recserver.sh
****************************************************
If the server is already running, the port won't be available, and sometimes there are some
zombie processes leftover after a crash. It can be useful to `sudo killall python` before launching.
See the commands in the shell script for more details, like the name of the logfile.
Starting (or restarting) the rec server:
****************************************
-----------------
DF - Sep 21, 2020
Start the Python process in the background:
-------------------------------------------
./invest/scripts/recreation_server/launch_recserver_twitter.sh
installed natcap.invest from github.com/natcap/invest:release/3.9 to get the latest server
bugfixes related to issue #304.
Setting up a new VM:
--------------------
The backend of the recreation model typically runs on a GCE VM.
Refer to invest/scripts/recreation_server/setup_vm.sh for VM setup.
--------------
DF - Sep, 2020
For a while we had two servers up, one running python3 and one running python27 for compatibility
with old invest clients. That's why there are two cache directories. For a few months we have
only had the python3 server running and no one has complained.
-----------------
DF - Oct 17, 2019
Updated the invest-davemfish/invest-env-py36 environment to branch feature/INVEST-3923-Migrate-to-Python37
The recmodel_server code running here is cross-compatible with python36, so I didn't bother creating a
new 3.7 env.
-----------------
DF - July 8, 2019
We're doing python 3 updates.
./invest-davemfish is a src tree with branch bugfix/PYTHON3-3895-27to36-compatibility
./invest-davemfish/invest-env-py36/ is a python3 env created with conda with the above branch installed.
('conda' should be available to all users.
e.g. conda activate /usr/local/recreation_server/invest-davemfish/invest-env-py36
We launched a rec server from that environment on port 54322.
It will be live alongside the python 2 server that's already on port 54321
They share data including the input CSV table and the recserver_cache_2017/
The invest bugfix/PYTHON3-3895-27to36-compatibility client source defaults to port 54322
Once this branch has been merged and released, we can kill the python 2 server on 54321.
The new port also required:
data.naturalcapitalproject.org/server_registry/invest_recreation_model_py36/index.html
----------------
DF - July 2 2018
I'm building the cache from the 2005-2017 table. I used launch_2017.sh with recserver_cache_2017 as the cache directory, and 55555 as the port.
The idea is to build the cache/quadtree without disrupting the existing server running on port 54321.
Then after the new cache is built:
* kill the 'dummy' server on 55555
* kill the 'real' server on 54321
* rename the 2017 cache dir back to recserver_cache
* relaunch a server with the 2005-2017 table on port 54321.
code should recognize that the quadtree already exists and skip that long process.
And for more details,
https://github.com/natcap/invest/wiki/Recreation-model-cloud-infrastructure

52
scripts/recreation_server/setup_vm.sh Normal file
View File

@ -0,0 +1,52 @@
# This file is meant to be a record of all the steps needed
# to setup a GCS VM running the recmodel_server.py
# It's adviseable to run these commands individually rather
# than execute this script, since that has not been tested.
# And because some of these steps might be interactive.
# Install GCS Fuse
export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
sudo apt-get update
sudo apt-get install gcsfuse git gcc g++
cd ~
curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
bash Miniforge3-$(uname)-$(uname -m).sh
mamba init && source .bashrc
cd /usr/local/recreation-server
mkdir invest_3_15_0
cd invest_3_15_0
git clone https://github.com/natcap/invest.git
cd invest
git checkout 3.15.0
mamba create -p ./env python=3.12
mamba activate ./env
mamba install "gdal>=3.4.2" "pygeoprocessing>=2.4.6" "numpy>=2.0"
pip install .
# Mount GCS Fuse
cd /usr/local/recreation-server/invest_3_15_0 && mkdir server && mkdir server/volume
gcsfuse --implicit-dirs -o ro natcap-recreation server/volume
# Listing all contents should build some indices and improve performance later
ls -R server/volume
# Start the recmodel_server process
# Review or update invest/scripts/recreation_server/execute_recmodel_server.py
# Then launch the Ptyhon process in the background:
chmod 755 invest/scripts/recreation_server/launch_recserver.sh
./invest/scripts/recreation_server/launch_recserver.sh
# Observe the server logfile to see if started up:
tail -f server/log.txt
# After setting up a new VM or new server cache,
# initialize a cron job that periodically clears out the cache.
# Copy the script so that updates to the invest repo don't clobber it
# Edit the paths to workspaces referenced in the script if needed.
cp invest/scripts/recreation_server/cron_find_rm_cached_workspaces.sh server/cron_find_rm_cached_workspaces.sh
chmod 755 server/cron_find_rm_cached_workspaces.sh
sudo crontab -e
# Enter: @daily /usr/local/recreation-server/invest_3_15_0/server/cron_find_rm_cached_workspaces.sh

34
setup.py
View File

@ -15,13 +15,26 @@ _REQUIREMENTS = [req.split(';')[0].split('#')[0].strip() for req in
if (not req.startswith(('#', 'hg+', 'git+'))
and len(req.strip()) > 0)]
# Since OSX Mavericks, the stdlib has been renamed. So if we're on OSX, we
# need to be sure to define which standard c++ library to use. I don't have
# access to a pre-Mavericks mac, so hopefully this won't break on someone's
# older system. Tested and it works on Mac OSX Catalina.
compiler_and_linker_args = []
if platform.system() == 'Darwin':
compiler_and_linker_args = ['-stdlib=libc++']
include_dirs = [numpy.get_include(), 'src/natcap/invest/managed_raster']
if platform.system() == 'Windows':
compiler_args = ['/std:c++20']
compiler_and_linker_args = []
if 'NATCAP_INVEST_GDAL_LIB_PATH' not in os.environ:
raise RuntimeError(
'env variable NATCAP_INVEST_GDAL_LIB_PATH is not defined. '
'This env variable is required when building on Windows. If '
'using conda to manage your gdal installation, you may set '
'NATCAP_INVEST_GDAL_LIB_PATH="$CONDA_PREFIX/Library".')
library_dirs = [f'{os.environ["NATCAP_INVEST_GDAL_LIB_PATH"]}/lib']
include_dirs.append(f'{os.environ["NATCAP_INVEST_GDAL_LIB_PATH"]}/include')
else:
compiler_args = [subprocess.run(
['gdal-config', '--cflags'], capture_output=True, text=True
).stdout.strip()]
compiler_and_linker_args = ['-std=c++20']
library_dirs = [subprocess.run(
['gdal-config', '--libs'], capture_output=True, text=True
).stdout.split()[0][2:]] # get the first argument which is the library path
class build_py(_build_py):
@ -50,11 +63,14 @@ setup(
Extension(
name=f'natcap.invest.{package}.{module}',
sources=[f'src/natcap/invest/{package}/{module}.pyx'],
extra_compile_args=compiler_args + compiler_and_linker_args,
include_dirs=include_dirs,
extra_compile_args=compiler_args + package_compiler_args + compiler_and_linker_args,
extra_link_args=compiler_and_linker_args,
language='c++',
libraries=['gdal'],
library_dirs=library_dirs,
define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")]
) for package, module, compiler_args in [
) for package, module, package_compiler_args in [
('delineateit', 'delineateit_core', []),
('recreation', 'out_of_core_quadtree', []),
# clang-14 defaults to -ffp-contract=on, which causes the

0
src/natcap/__init__.pxd Normal file
View File

10
src/natcap/invest/__init__.py
View File

@ -4,10 +4,8 @@ import logging
import os
import sys
from gettext import translation
import warnings
import babel
from osgeo import gdal
LOGGER = logging.getLogger('natcap.invest')
LOGGER.addHandler(logging.NullHandler())
@ -34,14 +32,6 @@ LOCALE_NAME_MAP = {
locale: babel.Locale(locale).display_name for locale in LOCALES
}
if not gdal.GetUseExceptions():
warnings.warn(('''
natcap.invest requires GDAL exceptions to be enabled. You must
call gdal.UseExceptions() to avoid unexpected behavior from
natcap.invest. A future version will enable exceptions on import.
gdal.UseExceptions() affects global state, so this may affect the
behavior of other packages.'''), FutureWarning)
def set_locale(locale_code):
"""Set the `gettext` attribute of natcap.invest.

1
src/natcap/invest/annual_water_yield.py
View File

@ -106,6 +106,7 @@ WATERSHED_OUTPUT_FIELDS = {
}
MODEL_SPEC = {
"model_id": "annual_water_yield",
"model_name": MODEL_METADATA["annual_water_yield"].model_title,
"pyname": MODEL_METADATA["annual_water_yield"].pyname,
"userguide": MODEL_METADATA["annual_water_yield"].userguide,

339
src/natcap/invest/carbon.py
View File

@ -4,7 +4,6 @@ import codecs
import logging
import os
import time
from functools import reduce
from osgeo import gdal
import numpy
@ -27,7 +26,7 @@ CARBON_OUTPUTS = {
"scenario, mapped from the Carbon Pools table to the LULC."),
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
} for pool, pool_name in [
('above', 'aboveground'),
@ -35,79 +34,54 @@ CARBON_OUTPUTS = {
('soil', 'soil'),
('dead', 'dead matter')
] for scenario, scenario_name in [
('cur', 'current'),
('fut', 'future'),
('redd', 'REDD')
('bas', 'baseline'),
('alt', 'alternate')
]
}
MODEL_SPEC = {
"model_id": "carbon",
"model_name": MODEL_METADATA["carbon"].model_title,
"pyname": MODEL_METADATA["carbon"].pyname,
"userguide": MODEL_METADATA["carbon"].userguide,
"args_with_spatial_overlap": {
"spatial_keys": ["lulc_cur_path", "lulc_fut_path", "lulc_redd_path"],
"spatial_keys": ["lulc_bas_path", "lulc_alt_path"],
},
"args": {
"workspace_dir": spec_utils.WORKSPACE,
"results_suffix": spec_utils.SUFFIX,
"n_workers": spec_utils.N_WORKERS,
"lulc_cur_path": {
"lulc_bas_path": {
**spec_utils.LULC,
"projected": True,
"projection_units": u.meter,
"about": gettext(
"A map of LULC for the current scenario. "
"All values in this raster must have corresponding "
"entries in the Carbon Pools table."),
"name": gettext("current LULC")
"A map of LULC for the baseline scenario, which must occur "
"prior to the alternate scenario. All values in this raster "
"must have corresponding entries in the Carbon Pools table."),
"name": gettext("baseline LULC")
},
"calc_sequestration": {
"type": "boolean",
"required": "do_valuation | do_redd",
"required": "do_valuation",
"about": gettext(
"Run sequestration analysis. This requires inputs "
"of LULC maps for both current and future "
"scenarios. Required if REDD scenario analysis or "
"run valuation model is selected."),
"of LULC maps for both baseline and alternate "
"scenarios. Required if run valuation model is selected."),
"name": gettext("calculate sequestration")
},
"lulc_fut_path": {
"lulc_alt_path": {
**spec_utils.LULC,
"projected": True,
"projection_units": u.meter,
"required": "calc_sequestration",
"about": gettext(
"A map of LULC for the future scenario. "
"If run valuation model is "
"selected, this should be the reference, or baseline, future "
"scenario against which to compare the REDD policy scenario. "
"All values in this raster must have corresponding entries in "
"the Carbon Pools table. Required if Calculate Sequestration "
"is selected."),
"name": gettext("future LULC")
},
"do_redd": {
"type": "boolean",
"required": False,
"about": gettext(
"Run REDD scenario analysis. This requires three "
"LULC maps: one for the current scenario, one "
"for the future baseline scenario, and one for the future "
"REDD policy scenario."),
"name": gettext("REDD scenario analysis")
},
"lulc_redd_path": {
**spec_utils.LULC,
"projected": True,
"projection_units": u.meter,
"required": "do_redd",
"about": gettext(
"A map of LULC for the REDD policy scenario. "
"All values in this raster must have corresponding entries in "
"the Carbon Pools table. Required if REDD Scenario Analysis "
"is selected."),
"name": gettext("REDD LULC")
"A map of LULC for the alternate scenario, which must occur "
"after the baseline scenario. All values in this raster must "
"have corresponding entries in the Carbon Pools table. "
"This raster must align with the Baseline LULC raster. "
"Required if Calculate Sequestration is selected."),
"name": gettext("alternate LULC")
},
"carbon_pools_path": {
"type": "csv",
@ -136,33 +110,34 @@ MODEL_SPEC = {
"that LULC type."),
"name": gettext("carbon pools")
},
"lulc_cur_year": {
"lulc_bas_year": {
"expression": "float(value).is_integer()",
"type": "number",
"units": u.year_AD,
"required": "do_valuation",
"about": gettext(
"The calendar year of the current scenario depicted in the "
"current LULC map. Required if Run Valuation model is selected."),
"name": gettext("current LULC year")
"The calendar year of the baseline scenario depicted in the "
"baseline LULC map. Must be < alternate LULC year. Required "
"if Run Valuation model is selected."),
"name": gettext("baseline LULC year")
},
"lulc_fut_year": {
"lulc_alt_year": {
"expression": "float(value).is_integer()",
"type": "number",
"units": u.year_AD,
"required": "do_valuation",
"about": gettext(
"The calendar year of the future scenario depicted in the "
"future LULC map. Required if Run Valuation model is selected."),
"name": f"future LULC year"
"The calendar year of the alternate scenario depicted in the "
"alternate LULC map. Must be > baseline LULC year. Required "
"if Run Valuation model is selected."),
"name": gettext("alternate LULC year")
},
"do_valuation": {
"type": "boolean",
"required": False,
"about": gettext(
"Calculate net present value for the future scenario, and the "
"REDD scenario if provided, and report it in the final HTML "
"document."),
"Calculate net present value for the alternate scenario "
"and report it in the final HTML document."),
"name": gettext("run valuation model")
},
"price_per_metric_ton_of_c": {
@ -175,20 +150,21 @@ MODEL_SPEC = {
"name": gettext("price of carbon")
},
"discount_rate": {
"type": "ratio",
"type": "percent",
"required": "do_valuation",
"about": gettext(
"The annual market discount rate in the price of carbon, "
"which reflects society's preference for immediate benefits "
"over future benefits. Required if Run Valuation model is "
"selected."),
"selected. This assumes that the baseline scenario is current "
"and the alternate scenario is in the future."),
"name": gettext("annual market discount rate")
},
"rate_change": {
"type": "ratio",
"type": "percent",
"required": "do_valuation",
"about": gettext(
"The relative annual increase of the price of carbon. "
"The relative annual change of the price of carbon. "
"Required if Run Valuation model is selected."),
"name": gettext("annual price change")
}
@ -197,60 +173,36 @@ MODEL_SPEC = {
"report.html": {
"about": "This file presents a summary of all data computed by the model. It also includes descriptions of all other output files produced by the model, so it is a good place to begin exploring and understanding model results. Because this is an HTML file, it can be opened with any web browser."
},
"tot_c_cur.tif": {
"about": "Raster showing the amount of carbon stored in each pixel for the current scenario. It is a sum of all of the carbon pools provided by the biophysical table.",
"c_storage_bas.tif": {
"about": "Raster showing the amount of carbon stored in each pixel for the baseline scenario. It is a sum of all of the carbon pools provided by the biophysical table.",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"tot_c_fut.tif": {
"about": "Raster showing the amount of carbon stored in each pixel for the future scenario. It is a sum of all of the carbon pools provided by the biophysical table.",
"c_storage_alt.tif": {
"about": "Raster showing the amount of carbon stored in each pixel for the alternate scenario. It is a sum of all of the carbon pools provided by the biophysical table.",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}},
"created_if": "lulc_fut_path"
"created_if": "lulc_alt_path"
},
"tot_c_redd.tif": {
"about": "Raster showing the amount of carbon stored in each pixel for the REDD scenario. It is a sum of all of the carbon pools provided by the biophysical table.",
"c_change_bas_alt.tif": {
"about": "Raster showing the difference in carbon stored between the alternate landscape and the baseline landscape. In this map some values may be negative and some positive. Positive values indicate sequestered carbon, negative values indicate carbon that was lost.",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}},
"created_if": "lulc_redd_path"
"created_if": "lulc_alt_path"
},
"delta_cur_fut.tif": {
"about": "Raster showing the difference in carbon stored between the future landscape and the current landscape. In this map some values may be negative and some positive. Positive values indicate sequestered carbon, negative values indicate carbon that was lost.",
"npv_alt.tif": {
"about": "Rasters showing the economic value of carbon sequestered between the baseline and the alternate landscape dates.",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.currency/u.hectare
}},
"created_if": "lulc_fut_path"
},
"delta_cur_redd.tif": {
"about": "Raster showing the difference in carbon stored between the REDD landscape and the current landscape. In this map some values may be negative and some positive. Positive values indicate sequestered carbon, negative values indicate carbon that was lost.",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
}},
"created_if": "lulc_redd_path"
},
"npv_fut.tif": {
"about": "Rasters showing the economic value of carbon sequestered between the current and the future landscape dates.",
"bands": {1: {
"type": "number",
"units": u.currency/u.pixel
}},
"created_if": "lulc_fut_path"
},
"npv_redd.tif": {
"about": "Rasters showing the economic value of carbon sequestered between the current and the REDD landscape dates.",
"bands": {1: {
"type": "number",
"units": u.currency/u.pixel
}},
"created_if": "lulc_redd_path"
"created_if": "lulc_alt_path"
},
"intermediate_outputs": {
"type": "directory",
@ -263,79 +215,61 @@ MODEL_SPEC = {
}
_OUTPUT_BASE_FILES = {
'tot_c_cur': 'tot_c_cur.tif',
'tot_c_fut': 'tot_c_fut.tif',
'tot_c_redd': 'tot_c_redd.tif',
'delta_cur_fut': 'delta_cur_fut.tif',
'delta_cur_redd': 'delta_cur_redd.tif',
'npv_fut': 'npv_fut.tif',
'npv_redd': 'npv_redd.tif',
'c_storage_bas': 'c_storage_bas.tif',
'c_storage_alt': 'c_storage_alt.tif',
'c_change_bas_alt': 'c_change_bas_alt.tif',
'npv_alt': 'npv_alt.tif',
'html_report': 'report.html',
}
_INTERMEDIATE_BASE_FILES = {
'c_above_cur': 'c_above_cur.tif',
'c_below_cur': 'c_below_cur.tif',
'c_soil_cur': 'c_soil_cur.tif',
'c_dead_cur': 'c_dead_cur.tif',
'c_above_fut': 'c_above_fut.tif',
'c_below_fut': 'c_below_fut.tif',
'c_soil_fut': 'c_soil_fut.tif',
'c_dead_fut': 'c_dead_fut.tif',
'c_above_redd': 'c_above_redd.tif',
'c_below_redd': 'c_below_redd.tif',
'c_soil_redd': 'c_soil_redd.tif',
'c_dead_redd': 'c_dead_redd.tif',
}
_TMP_BASE_FILES = {
'aligned_lulc_cur_path': 'aligned_lulc_cur.tif',
'aligned_lulc_fut_path': 'aligned_lulc_fut.tif',
'aligned_lulc_redd_path': 'aligned_lulc_redd.tif',
'c_above_bas': 'c_above_bas.tif',
'c_below_bas': 'c_below_bas.tif',
'c_soil_bas': 'c_soil_bas.tif',
'c_dead_bas': 'c_dead_bas.tif',
'c_above_alt': 'c_above_alt.tif',
'c_below_alt': 'c_below_alt.tif',
'c_soil_alt': 'c_soil_alt.tif',
'c_dead_alt': 'c_dead_alt.tif',
}
# -1.0 since carbon stocks are 0 or greater
_CARBON_NODATA = -1.0
def execute(args):
"""Carbon.
Calculate the amount of carbon stocks given a landscape, or the difference
due to a future change, and/or the tradeoffs between that and a REDD
scenario, and calculate economic valuation on those scenarios.
due to some change, and calculate economic valuation on those scenarios.
The model can operate on a single scenario, a combined present and future
scenario, as well as an additional REDD scenario.
The model can operate on a single scenario or a combined baseline and
alternate scenario.
Args:
args['workspace_dir'] (string): a path to the directory that will
write output and other temporary files during calculation.
args['results_suffix'] (string): appended to any output file name.
args['lulc_cur_path'] (string): a path to a raster representing the
current carbon stocks.
args['lulc_bas_path'] (string): a path to a raster representing the
baseline carbon stocks.
args['calc_sequestration'] (bool): if true, sequestration should
be calculated and 'lulc_fut_path' and 'do_redd' should be defined.
args['lulc_fut_path'] (string): a path to a raster representing future
be calculated and 'lulc_alt_path' should be defined.
args['lulc_alt_path'] (string): a path to a raster representing alternate
landcover scenario. Optional, but if present and well defined
will trigger a sequestration calculation.
args['do_redd'] ( bool): if true, REDD analysis should be calculated
and 'lulc_redd_path' should be defined
args['lulc_redd_path'] (string): a path to a raster representing the
alternative REDD scenario which is only possible if the
args['lulc_fut_path'] is present and well defined.
args['carbon_pools_path'] (string): path to CSV or that indexes carbon
storage density to lulc codes. (required if 'do_uncertainty' is
false)
args['lulc_cur_year'] (int/string): an integer representing the year
of `args['lulc_cur_path']` used if `args['do_valuation']`
args['lulc_bas_year'] (int/string): an integer representing the year
of `args['lulc_bas_path']` used if `args['do_valuation']`
is True.
args['lulc_fut_year'](int/string): an integer representing the year
of `args['lulc_fut_path']` used in valuation if it exists.
args['lulc_alt_year'](int/string): an integer representing the year
of `args['lulc_alt_path']` used in valuation if it exists.
Required if `args['do_valuation']` is True and
`args['lulc_fut_path']` is present and well defined.
`args['lulc_alt_path']` is present and well defined.
args['do_valuation'] (bool): if true then run the valuation model on
available outputs. Calculate NPV for a future scenario or a REDD
scenario and report in final HTML document.
available outputs. Calculate NPV for an alternate scenario and
report in final HTML document.
args['price_per_metric_ton_of_c'] (float): Is the present value of
carbon per metric ton. Used if `args['do_valuation']` is present
and True.
@ -361,8 +295,15 @@ def execute(args):
LOGGER.info('Building file registry')
file_registry = utils.build_file_registry(
[(_OUTPUT_BASE_FILES, output_dir),
(_INTERMEDIATE_BASE_FILES, intermediate_output_dir),
(_TMP_BASE_FILES, output_dir)], file_suffix)
(_INTERMEDIATE_BASE_FILES, intermediate_output_dir),], file_suffix)
if args['do_valuation'] and args['lulc_bas_year'] >= args['lulc_alt_year']:
raise ValueError(
"Invalid input for lulc_bas_year or lulc_alt_year. The Alternate "
f"LULC Year ({args['lulc_alt_year']}) must be greater than the "
f"Baseline LULC Year ({args['lulc_bas_year']}). Ensure that the "
"Baseline LULC Year is earlier than the Alternate LULC Year."
)
carbon_pool_df = validation.get_validated_dataframe(
args['carbon_pools_path'], **MODEL_SPEC['args']['carbon_pools_path'])
@ -383,7 +324,7 @@ def execute(args):
valid_scenarios = []
tifs_to_summarize = set() # passed to _generate_report()
for scenario_type in ['cur', 'fut', 'redd']:
for scenario_type in ['bas', 'alt']:
lulc_key = "lulc_%s_path" % (scenario_type)
if lulc_key in args and args[lulc_key]:
raster_info = pygeoprocessing.get_raster_info(args[lulc_key])
@ -427,7 +368,7 @@ def execute(args):
storage_path_list.append(file_registry[storage_key])
carbon_map_task_lookup[scenario_type].append(carbon_map_task)
output_key = 'tot_c_' + scenario_type
output_key = 'c_storage_' + scenario_type
LOGGER.info(
"Calculate carbon storage for '%s'", output_key)
@ -446,26 +387,24 @@ def execute(args):
# calculate sequestration
diff_rasters_task_lookup = {}
for scenario_type in ['fut', 'redd']:
if scenario_type not in valid_scenarios:
continue
output_key = 'delta_cur_' + scenario_type
if 'alt' in valid_scenarios:
output_key = 'c_change_bas_alt'
LOGGER.info("Calculate sequestration scenario '%s'", output_key)
diff_rasters_task = graph.add_task(
func=pygeoprocessing.raster_map,
kwargs=dict(
op=numpy.subtract, # delta = scenario C - current C
rasters=[file_registry['tot_c_' + scenario_type],
file_registry['tot_c_cur']],
op=numpy.subtract, # c_change = scenario C - baseline C
rasters=[file_registry['c_storage_alt'],
file_registry['c_storage_bas']],
target_path=file_registry[output_key],
target_nodata=_CARBON_NODATA),
target_path_list=[file_registry[output_key]],
dependent_task_list=[
sum_rasters_task_lookup['cur'],
sum_rasters_task_lookup[scenario_type]],
sum_rasters_task_lookup['bas'],
sum_rasters_task_lookup['alt']],
task_name='diff_rasters_for_%s' % output_key)
diff_rasters_task_lookup[scenario_type] = diff_rasters_task
diff_rasters_task_lookup['alt'] = diff_rasters_task
tifs_to_summarize.add(file_registry[output_key])
# calculate net present value
@ -473,22 +412,20 @@ def execute(args):
if 'do_valuation' in args and args['do_valuation']:
LOGGER.info('Constructing valuation formula.')
valuation_constant = _calculate_valuation_constant(
int(args['lulc_cur_year']), int(args['lulc_fut_year']),
int(args['lulc_bas_year']), int(args['lulc_alt_year']),
float(args['discount_rate']), float(args['rate_change']),
float(args['price_per_metric_ton_of_c']))
for scenario_type in ['fut', 'redd']:
if scenario_type not in valid_scenarios:
continue
output_key = 'npv_%s' % scenario_type
LOGGER.info("Calculating NPV for scenario '%s'", output_key)
if 'alt' in valid_scenarios:
output_key = 'npv_alt'
LOGGER.info("Calculating NPV for scenario 'alt'")
calculate_npv_task = graph.add_task(
_calculate_npv,
args=(file_registry['delta_cur_%s' % scenario_type],
args=(file_registry['c_change_bas_alt'],
valuation_constant, file_registry[output_key]),
target_path_list=[file_registry[output_key]],
dependent_task_list=[diff_rasters_task_lookup[scenario_type]],
dependent_task_list=[diff_rasters_task_lookup['alt']],
task_name='calculate_%s' % output_key)
calculate_npv_tasks.append(calculate_npv_task)
tifs_to_summarize.add(file_registry[output_key])
@ -505,16 +442,6 @@ def execute(args):
task_name='generate_report')
graph.join()
for tmp_filename_key in _TMP_BASE_FILES:
try:
tmp_filename = file_registry[tmp_filename_key]
if os.path.exists(tmp_filename):
os.remove(tmp_filename)
except OSError as os_error:
LOGGER.warning(
"Can't remove temporary file: %s\nOriginal Exception:\n%s",
file_registry[tmp_filename_key], os_error)
# element-wise sum function to pass to raster_map
def sum_op(*xs): return numpy.sum(xs, axis=0)
@ -541,17 +468,15 @@ def _generate_carbon_map(
Args:
lulc_path (string): landcover raster with integer pixels.
out_carbon_stock_path (string): path to output raster that will have
pixels with carbon storage values in them with units of Mg*C
pixels with carbon storage values in them with units of Mg/ha.
carbon_pool_by_type (dict): a dictionary that maps landcover values
to carbon storage densities per area (Mg C/Ha).
Returns:
None.
"""
lulc_info = pygeoprocessing.get_raster_info(lulc_path)
pixel_area = abs(numpy.prod(lulc_info['pixel_size']))
carbon_stock_by_type = dict([
(lulcid, stock * pixel_area / 10**4)
(lulcid, stock)
for lulcid, stock in carbon_pool_by_type.items()])
reclass_error_details = {
@ -563,22 +488,22 @@ def _generate_carbon_map(
def _calculate_valuation_constant(
lulc_cur_year, lulc_fut_year, discount_rate, rate_change,
lulc_bas_year, lulc_alt_year, discount_rate, rate_change,
price_per_metric_ton_of_c):
"""Calculate a net present valuation constant to multiply carbon storage.
Args:
lulc_cur_year (int): calendar year in present
lulc_fut_year (int): calendar year in future
lulc_bas_year (int): calendar year for baseline
lulc_alt_year (int): calendar year for alternate
discount_rate (float): annual discount rate as a percentage
rate_change (float): annual change in price of carbon as a percentage
price_per_metric_ton_of_c (float): currency amount of Mg of carbon
Returns:
a floating point number that can be used to multiply a delta carbon
storage value by to calculate NPV.
a floating point number that can be used to multiply a carbon
storage change value by to calculate NPV.
"""
n_years = lulc_fut_year - lulc_cur_year
n_years = lulc_alt_year - lulc_bas_year
ratio = (
1 / ((1 + discount_rate / 100) *
(1 + rate_change / 100)))
@ -596,11 +521,11 @@ def _calculate_valuation_constant(
return valuation_constant
def _calculate_npv(delta_carbon_path, valuation_constant, npv_out_path):
def _calculate_npv(c_change_carbon_path, valuation_constant, npv_out_path):
"""Calculate net present value.
Args:
delta_carbon_path (string): path to change in carbon storage over
c_change_carbon_path (string): path to change in carbon storage over
time.
valuation_constant (float): value to multiply each carbon storage
value by to calculate NPV.
@ -611,7 +536,7 @@ def _calculate_npv(delta_carbon_path, valuation_constant, npv_out_path):
"""
pygeoprocessing.raster_map(
op=lambda carbon: carbon * valuation_constant,
rasters=[delta_carbon_path],
rasters=[c_change_carbon_path],
target_path=npv_out_path)
@ -704,27 +629,33 @@ def _generate_report(raster_file_set, model_args, file_registry):
'<table><thead><tr><th>Description</th><th>Value</th><th>Units'
'</th><th>Raw File</th></tr></thead><tbody>')
carbon_units = 'metric tons'
# value lists are [sort priority, description, statistic, units]
report = [
(file_registry['tot_c_cur'], 'Total cur', 'Mg of C'),
(file_registry['tot_c_fut'], 'Total fut', 'Mg of C'),
(file_registry['tot_c_redd'], 'Total redd', 'Mg of C'),
(file_registry['delta_cur_fut'], 'Change in C for fut', 'Mg of C'),
(file_registry['delta_cur_redd'],
'Change in C for redd', 'Mg of C'),
(file_registry['npv_fut'],
'Net present value from cur to fut', 'currency units'),
(file_registry['npv_redd'],
'Net present value from cur to redd', 'currency units'),
(file_registry['c_storage_bas'], 'Baseline Carbon Storage',
carbon_units),
(file_registry['c_storage_alt'], 'Alternate Carbon Storage',
carbon_units),
(file_registry['c_change_bas_alt'], 'Change in Carbon Storage',
carbon_units),
(file_registry['npv_alt'],
'Net Present Value of Carbon Change', 'currency units'),
]
for raster_uri, description, units in report:
if raster_uri in raster_file_set:
summary_stat = _accumulate_totals(raster_uri)
total = _accumulate_totals(raster_uri)
raster_info = pygeoprocessing.get_raster_info(raster_uri)
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
# Since each pixel value is in Mg/ha, ``total`` is in (Mg/ha * px) = Mg•px/ha.
# Adjusted sum = ([total] Mg•px/ha) * ([pixel_area] m^2 / 1 px) * (1 ha / 10000 m^2) = Mg.
summary_stat = total * pixel_area / 10000
report_doc.write(
'<tr><td>%s</td><td class="number">%.2f</td><td>%s</td>'
'<td>%s</td></tr>' % (
description, summary_stat, units, raster_uri))
'<tr><td>%s</td><td class="number" data-summary-stat="%s">'
'%.2f</td><td>%s</td><td>%s</td></tr>' % (
description, description, summary_stat, units,
raster_uri))
report_doc.write('</tbody></table></body></html>')

22
src/natcap/invest/cli.py
View File

@ -12,15 +12,13 @@ import sys
import textwrap
import warnings
import natcap.invest
from natcap.invest import datastack
from natcap.invest import model_metadata
from natcap.invest import spec_utils
from natcap.invest import ui_server
from natcap.invest import utils
from pygeoprocessing.geoprocessing_core import GDALUseExceptions
with GDALUseExceptions():
import natcap.invest
from natcap.invest import datastack
from natcap.invest import model_metadata
from natcap.invest import set_locale
from natcap.invest import spec_utils
from natcap.invest import ui_server
from natcap.invest import utils
DEFAULT_EXIT_CODE = 1
LOGGER = logging.getLogger(__name__)
@ -470,6 +468,14 @@ def main(user_args=None):
# written to stdout if this exception is uncaught. This is by
# design.
model_module.execute(parsed_datastack.args)
LOGGER.info('Generating metadata for results')
try:
# If there's an exception from creating metadata
# I don't think we want to indicate a model failure
spec_utils.generate_metadata(model_module, parsed_datastack.args)
except Exception as exc:
LOGGER.warning(
'Something went wrong while generating metadata', exc_info=exc)
if args.subcommand == 'serve':
ui_server.app.run(port=args.port)

7
src/natcap/invest/coastal_blue_carbon/coastal_blue_carbon.py
View File

@ -161,6 +161,7 @@ INTERMEDIATE_DIR_NAME = 'intermediate'
OUTPUT_DIR_NAME = 'output'
MODEL_SPEC = {
"model_id": "coastal_blue_carbon",
"model_name": MODEL_METADATA["coastal_blue_carbon"].model_title,
"pyname": MODEL_METADATA["coastal_blue_carbon"].pyname,
"userguide": MODEL_METADATA["coastal_blue_carbon"].userguide,
@ -207,9 +208,9 @@ MODEL_SPEC = {
"biophysical_table_path": {
"name": gettext("biophysical table"),
"type": "csv",
"index_col": "code",
"index_col": "lucode",
"columns": {
"code": {
"lucode": {
"type": "integer",
"about": gettext(
"The LULC code that represents this LULC "
@ -1679,7 +1680,7 @@ def _track_disturbance(
disturbed_carbon_volume[:] = NODATA_FLOAT32_MIN
disturbed_carbon_volume[
~pygeoprocessing.array_equals_nodata(disturbance_magnitude_matrix,
NODATA_FLOAT32_MIN)] = 0.0
NODATA_FLOAT32_MIN)] = 0.0
if year_of_disturbance_band:
known_transition_years_matrix = (

13
src/natcap/invest/coastal_blue_carbon/preprocessor.py
View File

@ -22,6 +22,7 @@ BIOPHYSICAL_COLUMNS_SPEC = coastal_blue_carbon.MODEL_SPEC[
'args']['biophysical_table_path']['columns']
MODEL_SPEC = {
"model_id": "coastal_blue_carbon_preprocessor",
"model_name": MODEL_METADATA["coastal_blue_carbon_preprocessor"].model_title,
"pyname": MODEL_METADATA["coastal_blue_carbon_preprocessor"].pyname,
"userguide": MODEL_METADATA["coastal_blue_carbon_preprocessor"].userguide,
@ -36,9 +37,9 @@ MODEL_SPEC = {
"A table mapping LULC codes from the snapshot rasters to the "
"corresponding LULC class names, and whether or not the "
"class is a coastal blue carbon habitat."),
"index_col": "code",
"index_col": "lucode",
"columns": {
"code": {
"lucode": {
"type": "integer",
"about": gettext(
"LULC code. Every value in the "
@ -86,9 +87,9 @@ MODEL_SPEC = {
"index_col": "lulc-class",
"columns": {
"lulc-class": {
"type": "integer",
"type": "freestyle_string",
"about": gettext(
"LULC codes matching the codes in the biophysical "
"LULC class names matching those in the biophysical "
"table.")},
"[LULC]": {
"type": "option_string",
@ -114,7 +115,7 @@ MODEL_SPEC = {
"Table mapping each LULC type to impact and accumulation "
"information. This is a template that you will fill out to "
"create the biophysical table input to the main model."),
"index_col": "code",
"index_col": "lucode",
"columns": {
**BIOPHYSICAL_COLUMNS_SPEC,
# remove "expression" property which doesn't go in output spec
@ -394,7 +395,7 @@ def _create_biophysical_table(landcover_df, target_biophysical_table_path):
# have commas between fields.
row = []
for colname in target_column_names:
if colname == 'code':
if colname == 'lucode':
row.append(str(lulc_code))
else:
try:

1
src/natcap/invest/coastal_vulnerability.py
View File

@ -120,6 +120,7 @@ WWIII_FIELDS = {
}
MODEL_SPEC = {
"model_id": "coastal_vulnerability",
"model_name": MODEL_METADATA["coastal_vulnerability"].model_title,
"pyname": MODEL_METADATA["coastal_vulnerability"].pyname,
"userguide": MODEL_METADATA["coastal_vulnerability"].userguide,

74
src/natcap/invest/crop_production_percentile.py
View File

@ -242,6 +242,7 @@ nutrient_units = {
}
MODEL_SPEC = {
"model_id": "crop_production_percentile",
"model_name": MODEL_METADATA["crop_production_percentile"].model_title,
"pyname": MODEL_METADATA["crop_production_percentile"].pyname,
"userguide": MODEL_METADATA["crop_production_percentile"].userguide,
@ -761,7 +762,6 @@ def execute(args):
args['landcover_raster_path'],
interpolated_yield_percentile_raster_path,
crop_lucode,
pixel_area_ha,
percentile_crop_production_raster_path),
target_path_list=[percentile_crop_production_raster_path],
dependent_task_list=[
@ -836,7 +836,7 @@ def execute(args):
args=([(args['landcover_raster_path'], 1),
(interpolated_observed_yield_raster_path, 1),
(observed_yield_nodata, 'raw'), (landcover_nodata, 'raw'),
(crop_lucode, 'raw'), (pixel_area_ha, 'raw')],
(crop_lucode, 'raw')],
_mask_observed_yield_op, observed_production_raster_path,
gdal.GDT_Float32, observed_yield_nodata),
target_path_list=[observed_production_raster_path],
@ -853,7 +853,7 @@ def execute(args):
output_dir, 'result_table%s.csv' % file_suffix)
crop_names = crop_to_landcover_df.index.to_list()
tabulate_results_task = task_graph.add_task(
_ = task_graph.add_task(
func=tabulate_results,
args=(nutrient_df, yield_percentile_headers,
crop_names, pixel_area_ha,
@ -870,14 +870,14 @@ def execute(args):
output_dir, _AGGREGATE_VECTOR_FILE_PATTERN % (file_suffix))
aggregate_results_table_path = os.path.join(
output_dir, _AGGREGATE_TABLE_FILE_PATTERN % file_suffix)
aggregate_results_task = task_graph.add_task(
_ = task_graph.add_task(
func=aggregate_to_polygons,
args=(args['aggregate_polygon_path'],
target_aggregate_vector_path,
landcover_raster_info['projection_wkt'],
crop_names, nutrient_df,
yield_percentile_headers, output_dir, file_suffix,
aggregate_results_table_path),
yield_percentile_headers, pixel_area_ha, output_dir,
file_suffix, aggregate_results_table_path),
target_path_list=[target_aggregate_vector_path,
aggregate_results_table_path],
dependent_task_list=dependent_task_list,
@ -888,14 +888,14 @@ def execute(args):
def calculate_crop_production(lulc_path, yield_path, crop_lucode,
pixel_area_ha, target_path):
target_path):
"""Calculate crop production for a particular crop.
The resulting production value is:
- nodata, where either the LULC or yield input has nodata
- 0, where the LULC does not match the given LULC code
- yield * pixel area, where the given LULC code exists
- yield (in Mg/ha), where the given LULC code exists
Args:
lulc_path (str): path to a raster of LULC codes
@ -903,7 +903,6 @@ def calculate_crop_production(lulc_path, yield_path, crop_lucode,
by ``crop_lucode``, in units per hectare
crop_lucode (int): LULC code that identifies the crop of interest in
the ``lulc_path`` raster.
pixel_area_ha (number): Pixel area in hectares for both input rasters
target_path (str): Path to write the output crop production raster
Returns:
@ -911,7 +910,7 @@ def calculate_crop_production(lulc_path, yield_path, crop_lucode,
"""
pygeoprocessing.raster_map(
op=lambda lulc, _yield: numpy.where(
lulc == crop_lucode, _yield * pixel_area_ha, 0),
lulc == crop_lucode, _yield, 0),
rasters=[lulc_path, yield_path],
target_path=target_path,
target_nodata=_NODATA_YIELD)
@ -941,7 +940,7 @@ def _zero_observed_yield_op(observed_yield_array, observed_yield_nodata):
def _mask_observed_yield_op(
lulc_array, observed_yield_array, observed_yield_nodata,
landcover_nodata, crop_lucode, pixel_area_ha):
landcover_nodata, crop_lucode):
"""Mask total observed yield to crop lulc type.
Args:
@ -950,7 +949,6 @@ def _mask_observed_yield_op(
observed_yield_nodata (float): yield raster nodata value
landcover_nodata (float): landcover raster nodata value
crop_lucode (int): code used to mask in the current crop
pixel_area_ha (float): area of lulc raster cells (hectares)
Returns:
numpy.ndarray with float values of yields masked to crop_lucode
@ -959,13 +957,13 @@ def _mask_observed_yield_op(
result = numpy.empty(lulc_array.shape, dtype=numpy.float32)
if landcover_nodata is not None:
result[:] = observed_yield_nodata
valid_mask = ~pygeoprocessing.array_equals_nodata(lulc_array, landcover_nodata)
valid_mask = ~pygeoprocessing.array_equals_nodata(lulc_array,
landcover_nodata)
result[valid_mask] = 0
else:
result[:] = 0
lulc_mask = lulc_array == crop_lucode
result[lulc_mask] = (
observed_yield_array[lulc_mask] * pixel_area_ha)
result[lulc_mask] = observed_yield_array[lulc_mask]
return result
@ -986,7 +984,7 @@ def tabulate_results(
landcover_raster_path (string): path to landcover raster
landcover_nodata (float): landcover raster nodata value
output_dir (string): the file path to the output workspace.
file_suffix (string): string to appened to any output filenames.
file_suffix (string): string to append to any output filenames.
target_table_path (string): path to 'result_table.csv' in the output
workspace
@ -1007,6 +1005,11 @@ def tabulate_results(
for nutrient_id in _EXPECTED_NUTRIENT_TABLE_HEADERS
for yield_percentile_id in sorted(yield_percentile_headers) + [
'yield_observed']]
# Since pixel values in observed and percentile rasters are Mg/(ha•yr),
# raster sums are (Mg•px)/(ha•yr). Before recording sums in
# production_lookup dictionary, convert to Mg/yr by multiplying by ha/px.
with open(target_table_path, 'w') as result_table:
result_table.write(
'crop,area (ha),' + 'production_observed,' +
@ -1038,6 +1041,7 @@ def tabulate_results(
production_pixel_count += numpy.count_nonzero(
valid_mask & (yield_block > 0))
yield_sum += numpy.sum(yield_block[valid_mask])
yield_sum *= pixel_area_ha
production_area = production_pixel_count * pixel_area_ha
production_lookup['observed'] = yield_sum
result_table.write(',%f' % production_area)
@ -1055,6 +1059,7 @@ def tabulate_results(
yield_sum += numpy.sum(
yield_block[~pygeoprocessing.array_equals_nodata(
yield_block, _NODATA_YIELD)])
yield_sum *= pixel_area_ha
production_lookup[yield_percentile_id] = yield_sum
result_table.write(",%f" % yield_sum)
@ -1080,7 +1085,8 @@ def tabulate_results(
(landcover_raster_path, 1)):
if landcover_nodata is not None:
total_area += numpy.count_nonzero(
~pygeoprocessing.array_equals_nodata(band_values, landcover_nodata))
~pygeoprocessing.array_equals_nodata(band_values,
landcover_nodata))
else:
total_area += band_values.size
result_table.write(
@ -1090,8 +1096,8 @@ def tabulate_results(
def aggregate_to_polygons(
base_aggregate_vector_path, target_aggregate_vector_path,
landcover_raster_projection, crop_names,
nutrient_df, yield_percentile_headers, output_dir, file_suffix,
landcover_raster_projection, crop_names, nutrient_df,
yield_percentile_headers, pixel_area_ha, output_dir, file_suffix,
target_aggregate_table_path):
"""Write table with aggregate results of yield and nutrient values.
@ -1108,8 +1114,9 @@ def aggregate_to_polygons(
nutrient_df (pandas.DataFrame): a table of nutrient values by crop
yield_percentile_headers (list): list of strings indicating percentiles
at which yield was calculated.
pixel_area_ha (float): area of lulc raster cells (hectares)
output_dir (string): the file path to the output workspace.
file_suffix (string): string to appened to any output filenames.
file_suffix (string): string to append to any output filenames.
target_aggregate_table_path (string): path to 'aggregate_results.csv'
in the output workspace
@ -1124,6 +1131,10 @@ def aggregate_to_polygons(
target_aggregate_vector_path,
driver_name='ESRI Shapefile')
# Since pixel values are Mg/(ha•yr), zonal stats sum is (Mg•px)/(ha•yr).
# Before writing sum to results tables or when using sum to calculate
# nutrient yields, convert to Mg/yr by multiplying by ha/px.
# loop over every crop and query with pgp function
total_yield_lookup = {}
total_nutrient_table = collections.defaultdict(
@ -1153,11 +1164,12 @@ def aggregate_to_polygons(
crop_name, yield_percentile_id)]:
total_nutrient_table[nutrient_id][
yield_percentile_id][id_index] += (
nutrient_factor *
total_yield_lookup['%s_%s' % (
crop_name, yield_percentile_id)][
id_index]['sum'] *
nutrient_df[nutrient_id][crop_name])
nutrient_factor
* total_yield_lookup[
'%s_%s' % (crop_name, yield_percentile_id)
][id_index]['sum']
* pixel_area_ha
* nutrient_df[nutrient_id][crop_name])
# process observed
observed_yield_path = os.path.join(
@ -1171,10 +1183,11 @@ def aggregate_to_polygons(
for id_index in total_yield_lookup[f'{crop_name}_observed']:
total_nutrient_table[
nutrient_id]['observed'][id_index] += (
nutrient_factor *
total_yield_lookup[
f'{crop_name}_observed'][id_index]['sum'] *
nutrient_df[nutrient_id][crop_name])
nutrient_factor
* total_yield_lookup[
f'{crop_name}_observed'][id_index]['sum']
* pixel_area_ha
* nutrient_df[nutrient_id][crop_name])
# report everything to a table
with open(target_aggregate_table_path, 'w') as aggregate_table:
@ -1193,7 +1206,8 @@ def aggregate_to_polygons(
for id_index in list(total_yield_lookup.values())[0]:
aggregate_table.write('%s,' % id_index)
aggregate_table.write(','.join([
str(total_yield_lookup[yield_header][id_index]['sum'])
str(total_yield_lookup[yield_header][id_index]['sum']
* pixel_area_ha)
for yield_header in sorted(total_yield_lookup)]))
for nutrient_id in _EXPECTED_NUTRIENT_TABLE_HEADERS:

95
src/natcap/invest/crop_production_regression.py
View File

@ -1,4 +1,4 @@
"""InVEST Crop Production Percentile Model."""
"""InVEST Crop Production Regression Model."""
import collections
import logging
import os
@ -68,6 +68,7 @@ NUTRIENTS = [
]
MODEL_SPEC = {
"model_id": "crop_production_regression",
"model_name": MODEL_METADATA["crop_production_regression"].model_title,
"pyname": MODEL_METADATA["crop_production_regression"].pyname,
"userguide": MODEL_METADATA["crop_production_regression"].userguide,
@ -218,8 +219,8 @@ MODEL_SPEC = {
"about": f"{x} {name} production within the polygon",
"type": "number",
"units": units
} for nutrient, name, units in NUTRIENTS
for x in ["modeled", "observed"]
} for (nutrient, name, units) in NUTRIENTS
for x in ["modeled", "observed"]
}
}
},
@ -251,8 +252,8 @@ MODEL_SPEC = {
"about": f"{x} {name} production from the crop",
"type": "number",
"units": units
} for nutrient, name, units in NUTRIENTS
for x in ["modeled", "observed"]
} for (nutrient, name, units) in NUTRIENTS
for x in ["modeled", "observed"]
}
}
},
@ -660,8 +661,7 @@ def execute(args):
(regression_parameter_raster_path_lookup['c_n'], 1),
(args['landcover_raster_path'], 1),
(crop_to_fertilization_rate_df['nitrogen_rate'][crop_name],
'raw'),
(crop_lucode, 'raw'), (pixel_area_ha, 'raw')],
'raw'), (crop_lucode, 'raw')],
_x_yield_op,
nitrogen_yield_raster_path, gdal.GDT_Float32, _NODATA_YIELD),
target_path_list=[nitrogen_yield_raster_path],
@ -679,8 +679,7 @@ def execute(args):
(regression_parameter_raster_path_lookup['c_p2o5'], 1),
(args['landcover_raster_path'], 1),
(crop_to_fertilization_rate_df['phosphorus_rate'][crop_name],
'raw'),
(crop_lucode, 'raw'), (pixel_area_ha, 'raw')],
'raw'), (crop_lucode, 'raw')],
_x_yield_op,
phosphorus_yield_raster_path, gdal.GDT_Float32, _NODATA_YIELD),
target_path_list=[phosphorus_yield_raster_path],
@ -698,8 +697,7 @@ def execute(args):
(regression_parameter_raster_path_lookup['c_k2o'], 1),
(args['landcover_raster_path'], 1),
(crop_to_fertilization_rate_df['potassium_rate'][crop_name],
'raw'),
(crop_lucode, 'raw'), (pixel_area_ha, 'raw')],
'raw'), (crop_lucode, 'raw')],
_x_yield_op,
potassium_yield_raster_path, gdal.GDT_Float32, _NODATA_YIELD),
target_path_list=[potassium_yield_raster_path],
@ -794,7 +792,7 @@ def execute(args):
args=([(args['landcover_raster_path'], 1),
(interpolated_observed_yield_raster_path, 1),
(observed_yield_nodata, 'raw'), (landcover_nodata, 'raw'),
(crop_lucode, 'raw'), (pixel_area_ha, 'raw')],
(crop_lucode, 'raw')],
_mask_observed_yield_op, observed_production_raster_path,
gdal.GDT_Float32, observed_yield_nodata),
target_path_list=[observed_production_raster_path],
@ -834,10 +832,10 @@ def execute(args):
func=aggregate_regression_results_to_polygons,
args=(args['aggregate_polygon_path'],
target_aggregate_vector_path,
aggregate_results_table_path,
landcover_raster_info['projection_wkt'],
crop_names, nutrient_df,
output_dir, file_suffix,
aggregate_results_table_path),
crop_names, nutrient_df, pixel_area_ha,
output_dir, file_suffix),
target_path_list=[target_aggregate_vector_path,
aggregate_results_table_path],
dependent_task_list=dependent_task_list,
@ -848,12 +846,12 @@ def execute(args):
def _x_yield_op(
y_max, b_x, c_x, lulc_array, fert_rate, crop_lucode, pixel_area_ha):
y_max, b_x, c_x, lulc_array, fert_rate, crop_lucode):
"""Calc generalized yield op, Ymax*(1-b_NP*exp(-cN * N_GC)).
The regression model has identical mathematical equations for
the nitrogen, phosphorus, and potassium. The only difference is
the scalars in the equation (fertilization rate and pixel area).
the nitrogen, phosphorus, and potassium. The only difference is
the scalar in the equation (fertilization rate).
"""
result = numpy.empty(b_x.shape, dtype=numpy.float32)
result[:] = _NODATA_YIELD
@ -862,7 +860,7 @@ def _x_yield_op(
~pygeoprocessing.array_equals_nodata(b_x, _NODATA_YIELD) &
~pygeoprocessing.array_equals_nodata(c_x, _NODATA_YIELD) &
(lulc_array == crop_lucode))
result[valid_mask] = pixel_area_ha * y_max[valid_mask] * (
result[valid_mask] = y_max[valid_mask] * (
1 - b_x[valid_mask] * numpy.exp(
-c_x[valid_mask] * fert_rate))
@ -897,7 +895,7 @@ def _zero_observed_yield_op(observed_yield_array, observed_yield_nodata):
def _mask_observed_yield_op(
lulc_array, observed_yield_array, observed_yield_nodata,
landcover_nodata, crop_lucode, pixel_area_ha):
landcover_nodata, crop_lucode):
"""Mask total observed yield to crop lulc type.
Args:
@ -906,7 +904,6 @@ def _mask_observed_yield_op(
observed_yield_nodata (float): yield raster nodata value
landcover_nodata (float): landcover raster nodata value
crop_lucode (int): code used to mask in the current crop
pixel_area_ha (float): area of lulc raster cells (hectares)
Returns:
numpy.ndarray with float values of yields masked to crop_lucode
@ -915,13 +912,13 @@ def _mask_observed_yield_op(
result = numpy.empty(lulc_array.shape, dtype=numpy.float32)
if landcover_nodata is not None:
result[:] = observed_yield_nodata
valid_mask = ~pygeoprocessing.array_equals_nodata(lulc_array, landcover_nodata)
valid_mask = ~pygeoprocessing.array_equals_nodata(
lulc_array, landcover_nodata)
result[valid_mask] = 0
else:
result[:] = 0
lulc_mask = lulc_array == crop_lucode
result[lulc_mask] = (
observed_yield_array[lulc_mask] * pixel_area_ha)
result[lulc_mask] = observed_yield_array[lulc_mask]
return result
@ -952,6 +949,11 @@ def tabulate_regression_results(
nutrient_id + '_' + mode
for nutrient_id in _EXPECTED_NUTRIENT_TABLE_HEADERS
for mode in ['modeled', 'observed']]
# Since pixel values in observed and percentile rasters are Mg/(ha•yr),
# raster sums are (Mg•px)/(ha•yr). Before recording sums in
# production_lookup dictionary, convert to Mg/yr by multiplying by ha/px.
with open(target_table_path, 'w') as result_table:
result_table.write(
'crop,area (ha),' + 'production_observed,production_modeled,' +
@ -982,6 +984,7 @@ def tabulate_regression_results(
production_pixel_count += numpy.count_nonzero(
valid_mask & (yield_block > 0.0))
yield_sum += numpy.sum(yield_block[valid_mask])
yield_sum *= pixel_area_ha
production_area = production_pixel_count * pixel_area_ha
production_lookup['observed'] = yield_sum
result_table.write(',%f' % production_area)
@ -997,6 +1000,7 @@ def tabulate_regression_results(
# _NODATA_YIELD will always have a value (defined above)
yield_block[~pygeoprocessing.array_equals_nodata(
yield_block, _NODATA_YIELD)])
yield_sum *= pixel_area_ha
production_lookup['modeled'] = yield_sum
result_table.write(",%f" % yield_sum)
@ -1031,9 +1035,8 @@ def tabulate_regression_results(
def aggregate_regression_results_to_polygons(
base_aggregate_vector_path, target_aggregate_vector_path,
landcover_raster_projection, crop_names,
nutrient_df, output_dir, file_suffix,
target_aggregate_table_path):
aggregate_results_table_path, landcover_raster_projection,
crop_names, nutrient_df, pixel_area_ha, output_dir, file_suffix):
"""Write table with aggregate results of yield and nutrient values.
Use zonal statistics to summarize total observed and interpolated
@ -1042,15 +1045,16 @@ def aggregate_regression_results_to_polygons(
Args:
base_aggregate_vector_path (string): path to polygon vector
target_aggregate_vector_path (string):
path to re-projected copy of polygon vector
target_aggregate_vector_path (string): path to re-projected copy of
polygon vector
aggregate_results_table_path (string): path to CSV file where aggregate
results will be reported.
landcover_raster_projection (string): a WKT projection string
crop_names (list): list of crop names
nutrient_df (pandas.DataFrame): a table of nutrient values by crop
pixel_area_ha (float): area of lulc raster cells (hectares)
output_dir (string): the file path to the output workspace.
file_suffix (string): string to append to any output filenames.
target_aggregate_table_path (string): path to 'aggregate_results.csv'
in the output workspace
Returns:
None
@ -1062,6 +1066,10 @@ def aggregate_regression_results_to_polygons(
target_aggregate_vector_path,
driver_name='ESRI Shapefile')
# Since pixel values are Mg/(ha•yr), zonal stats sum is (Mg•px)/(ha•yr).
# Before writing sum to results tables or when using sum to calculate
# nutrient yields, convert to Mg/yr by multiplying by ha/px.
# loop over every crop and query with pgp function
total_yield_lookup = {}
total_nutrient_table = collections.defaultdict(
@ -1085,10 +1093,11 @@ def aggregate_regression_results_to_polygons(
for fid_index in total_yield_lookup['%s_modeled' % crop_name]:
total_nutrient_table[nutrient_id][
'modeled'][fid_index] += (
nutrient_factor *
total_yield_lookup['%s_modeled' % crop_name][
fid_index]['sum'] *
nutrient_df[nutrient_id][crop_name])
nutrient_factor
* total_yield_lookup['%s_modeled' % crop_name][
fid_index]['sum']
* pixel_area_ha
* nutrient_df[nutrient_id][crop_name])
# process observed
observed_yield_path = os.path.join(
@ -1103,15 +1112,14 @@ def aggregate_regression_results_to_polygons(
'%s_observed' % crop_name]:
total_nutrient_table[
nutrient_id]['observed'][fid_index] += (
nutrient_factor * # percent crop used * 1000 [100g per Mg]
total_yield_lookup[
'%s_observed' % crop_name][fid_index]['sum'] *
nutrient_df[nutrient_id][crop_name]) # nutrient unit per 100g crop
nutrient_factor # percent crop used * 1000 [100g per Mg]
* total_yield_lookup[
'%s_observed' % crop_name][fid_index]['sum']
* pixel_area_ha
* nutrient_df[nutrient_id][crop_name]) # nutrient unit per 100g crop
# report everything to a table
aggregate_table_path = os.path.join(
output_dir, _AGGREGATE_TABLE_FILE_PATTERN % file_suffix)
with open(aggregate_table_path, 'w') as aggregate_table:
with open(aggregate_results_table_path, 'w') as aggregate_table:
# write header
aggregate_table.write('FID,')
aggregate_table.write(','.join(sorted(total_yield_lookup)) + ',')
@ -1127,7 +1135,8 @@ def aggregate_regression_results_to_polygons(
for id_index in list(total_yield_lookup.values())[0]:
aggregate_table.write('%s,' % id_index)
aggregate_table.write(','.join([
str(total_yield_lookup[yield_header][id_index]['sum'])
str(total_yield_lookup[yield_header][id_index]['sum']
* pixel_area_ha)
for yield_header in sorted(total_yield_lookup)]))
for nutrient_id in _EXPECTED_NUTRIENT_TABLE_HEADERS:

1
src/natcap/invest/delineateit/delineateit.py
View File

@ -25,6 +25,7 @@ from . import delineateit_core
LOGGER = logging.getLogger(__name__)
MODEL_SPEC = {
"model_id": "delineateit",
"model_name": MODEL_METADATA["delineateit"].model_title,
"pyname": MODEL_METADATA["delineateit"].pyname,
"userguide": MODEL_METADATA["delineateit"].userguide,

38
src/natcap/invest/forest_carbon_edge_effect.py
View File

@ -7,7 +7,6 @@ import logging
import os
import pickle
import time
import uuid
import numpy
import pandas
@ -33,6 +32,7 @@ DISTANCE_UPPER_BOUND = 500e3
NODATA_VALUE = -1
MODEL_SPEC = {
"model_id": "forest_carbon_edge_effect",
"model_name": MODEL_METADATA["forest_carbon_edge_effect"].model_title,
"pyname": MODEL_METADATA["forest_carbon_edge_effect"].pyname,
"userguide": MODEL_METADATA["forest_carbon_edge_effect"].userguide,
@ -187,13 +187,13 @@ MODEL_SPEC = {
"outputs": {
"carbon_map.tif": {
"about": (
"A map of carbon stock per pixel, with the amount in forest derived from the regression based on "
"distance to forest edge, and the amount in non-forest classes according to the biophysical table. "
"Note that because the map displays carbon per pixel, coarser resolution maps should have higher "
"values for carbon, because the pixel areas are larger."),
"A map of carbon stock per hectare, with the amount in forest "
"derived from the regression based on distance to forest "
"edge, and the amount in non-forest classes according to the "
"biophysical table. "),
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"aggregated_carbon_stocks.shp": {
@ -205,7 +205,7 @@ MODEL_SPEC = {
"units": u.metric_ton,
"about": "Total carbon in the area."
},
"c_ha_mean":{
"c_ha_mean": {
"type": "number",
"units": u.metric_ton/u.hectare,
"about": "Mean carbon density in the area."
@ -580,17 +580,24 @@ def _aggregate_carbon_map(
target_aggregate_layer.ResetReading()
target_aggregate_layer.StartTransaction()
# Since pixel values are Mg/ha, raster sum is (Mg•px)/ha.
# To convert to Mg, multiply by ha/px.
raster_info = pygeoprocessing.get_raster_info(carbon_map_path)
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
ha_per_px = pixel_area / 10000
for poly_feat in target_aggregate_layer:
poly_fid = poly_feat.GetFID()
poly_feat.SetField(
'c_sum', float(serviceshed_stats[poly_fid]['sum']))
'c_sum', float(serviceshed_stats[poly_fid]['sum'] * ha_per_px))
# calculates mean pixel value per ha in for each feature in AOI
poly_geom = poly_feat.GetGeometryRef()
poly_area_ha = poly_geom.GetArea() / 1e4 # converts m^2 to hectare
poly_geom = None
poly_feat.SetField(
'c_ha_mean',
float(serviceshed_stats[poly_fid]['sum'] / poly_area_ha))
float(serviceshed_stats[poly_fid]['sum'] / poly_area_ha
* ha_per_px))
target_aggregate_layer.SetFeature(poly_feat)
target_aggregate_layer.CommitTransaction()
@ -629,9 +636,6 @@ def _calculate_lulc_carbon_map(
biophysical_table_path, **MODEL_SPEC['args']['biophysical_table_path'])
lucode_to_per_cell_carbon = {}
cell_size = pygeoprocessing.get_raster_info(
lulc_raster_path)['pixel_size'] # in meters
cell_area_ha = abs(cell_size[0]) * abs(cell_size[1]) / 10000
# Build a lookup table
for lucode, row in biophysical_df.iterrows():
@ -648,8 +652,7 @@ def _calculate_lulc_carbon_map(
"Could not interpret carbon pool value as a number. "
f"lucode: {lucode}, pool_type: {carbon_pool_type}, "
f"value: {row[carbon_pool_type]}")
lucode_to_per_cell_carbon[lucode] = row[carbon_pool_type] * cell_area_ha
lucode_to_per_cell_carbon[lucode] = row[carbon_pool_type]
# map aboveground carbon from table to lulc that is not forest
reclass_error_details = {
@ -873,7 +876,6 @@ def _calculate_tropical_forest_edge_carbon_map(
cell_xsize, cell_ysize = pygeoprocessing.get_raster_info(
edge_distance_path)['pixel_size']
cell_size_km = (abs(cell_xsize) + abs(cell_ysize))/2 / 1000
cell_area_ha = (abs(cell_xsize) * abs(cell_ysize)) / 10000
# Loop memory block by memory block, calculating the forest edge carbon
# for every forest pixel.
@ -957,19 +959,19 @@ def _calculate_tropical_forest_edge_carbon_map(
biomass[mask_1] = (
thetas[mask_1][:, 0] - thetas[mask_1][:, 1] * numpy.exp(
-thetas[mask_1][:, 2] * valid_edge_distances_km[mask_1])
) * cell_area_ha
)
# logarithmic model
# biomass_2 = t1 + t2 * numpy.log(edge_dist_km)
biomass[mask_2] = (
thetas[mask_2][:, 0] + thetas[mask_2][:, 1] * numpy.log(
valid_edge_distances_km[mask_2])) * cell_area_ha
valid_edge_distances_km[mask_2]))
# linear regression
# biomass_3 = t1 + t2 * edge_dist_km
biomass[mask_3] = (
thetas[mask_3][:, 0] + thetas[mask_3][:, 1] *
valid_edge_distances_km[mask_3]) * cell_area_ha
valid_edge_distances_km[mask_3])
# reshape the array so that each set of points is in a separate
# dimension, here distances are distances to each valid model

191
src/natcap/invest/habitat_quality.py
View File

@ -26,8 +26,15 @@ MISSING_THREAT_RASTER_MSG = gettext(
"A threat raster for threats: {threat_list} was not found or it "
"could not be opened by GDAL.")
DUPLICATE_PATHS_MSG = gettext("Threat paths must be unique. Duplicates: ")
INVALID_MAX_DIST_MSG = gettext(
"The maximum distance value for threats: {threat_list} is less than "
"or equal to 0. MAX_DIST must be a positive value.")
MISSING_MAX_DIST_MSG = gettext(
"Maximum distance value is missing for threats: {threat_list}.")
MISSING_WEIGHT_MSG = gettext("Weight value is missing for threats: {threat_list}.")
MODEL_SPEC = {
"model_id": "habitat_quality",
"model_name": MODEL_METADATA["habitat_quality"].model_title,
"pyname": MODEL_METADATA["habitat_quality"].pyname,
"userguide": MODEL_METADATA["habitat_quality"].userguide,
@ -83,7 +90,7 @@ MODEL_SPEC = {
"corresponding column in the Sensitivity table.")},
"max_dist": {
"type": "number",
"units": u.kilometer,
"units": u.meter,
"about": gettext(
"The maximum distance over which each threat affects "
"habitat quality. The impact of each degradation "
@ -172,9 +179,9 @@ MODEL_SPEC = {
},
"sensitivity_table_path": {
"type": "csv",
"index_col": "lulc",
"index_col": "lucode",
"columns": {
"lulc": spec_utils.LULC_TABLE_COLUMN,
"lucode": spec_utils.LULC_TABLE_COLUMN,
"name": {
"type": "freestyle_string",
"required": False
@ -282,9 +289,9 @@ MODEL_SPEC = {
"rarity_c.csv": {
"about": ("Table of rarity values by LULC code for the "
"current landscape."),
"index_col": "lulc_code",
"index_col": "lucode",
"columns": {
"lulc_code": {
"lucode": {
"type": "number",
"units": u.none,
"about": "LULC class",
@ -314,9 +321,9 @@ MODEL_SPEC = {
"rarity_f.csv": {
"about": ("Table of rarity values by LULC code for the "
"future landscape."),
"index_col": "lulc_code",
"index_col": "lucode",
"columns": {
"lulc_code": {
"lucode": {
"type": "number",
"units": u.none,
"about": "LULC class",
@ -363,6 +370,10 @@ MODEL_SPEC = {
"filtered_[THREAT]_aligned.tif": {
"about": "Filtered threat raster",
"bands": {1: {"type": "ratio"}},
},
"degradation_[THREAT].tif": {
"about": "Degradation raster for each threat",
"bands": {1: {"type": "ratio"}},
}
}
},
@ -640,6 +651,7 @@ def execute(args):
threat_decay_task_list = []
sensitivity_task_list = []
individual_degradation_task_list = []
# Create raster of habitat based on habitat field
habitat_raster_path = os.path.join(
@ -657,12 +669,9 @@ def execute(args):
dependent_task_list=[align_task],
task_name=f'habitat_raster{lulc_key}')
# initialize a list that will store all the threat/threat rasters
# initialize a list that will store all the degradation rasters for each threat
# after they have been adjusted for distance, weight, and access
deg_raster_list = []
# a list to keep track of the normalized weight for each threat
weight_list = numpy.array([])
indiv_deg_raster_list = []
# variable to indicate whether we should break out of calculations
# for a land cover because a threat raster was not found
@ -736,29 +745,32 @@ def execute(args):
task_name=f'sens_raster_{row["decay"]}{lulc_key}_{threat}')
sensitivity_task_list.append(sens_threat_task)
# get the normalized weight for each threat
# get the normalized weight for the threat
# which is used to calculate threat degradation
weight_avg = row['weight'] / weight_sum
# add the threat raster adjusted by distance and the raster
# representing sensitivity to the list to be past to
# vectorized_rasters below
deg_raster_list.append(filtered_threat_raster_path)
deg_raster_list.append(sens_raster_path)
# Calculate degradation for each threat
indiv_threat_raster_path = os.path.join(
intermediate_output_dir, f'degradation_{threat}{lulc_key}{file_suffix}.tif')
# store the normalized weight for each threat in a list that
# will be used below in total_degradation
weight_list = numpy.append(weight_list, weight_avg)
individual_threat_task = task_graph.add_task(
func=_calculate_individual_degradation,
args=(filtered_threat_raster_path, sens_raster_path, weight_avg,
access_raster_path, indiv_threat_raster_path),
target_path_list=[indiv_threat_raster_path],
dependent_task_list=[
sens_threat_task, dist_decay_task,
*access_task_list],
task_name=f'deg raster {lulc_key} {threat}')
individual_degradation_task_list.append(individual_threat_task)
indiv_deg_raster_list.append(indiv_threat_raster_path)
# check to see if we got here because a threat raster was missing
# for baseline lulc, if so then we want to skip to the next landcover
if exit_landcover:
continue
# add the access_raster onto the end of the collected raster list. The
# access_raster will be values from the shapefile if provided or a
# raster filled with all 1's if not
deg_raster_list.append(access_raster_path)
deg_sum_raster_path = os.path.join(
output_dir, f'deg_sum{lulc_key}{file_suffix}.tif')
@ -766,11 +778,9 @@ def execute(args):
total_degradation_task = task_graph.add_task(
func=_calculate_total_degradation,
args=(deg_raster_list, deg_sum_raster_path, weight_list),
args=(indiv_deg_raster_list, deg_sum_raster_path),
target_path_list=[deg_sum_raster_path],
dependent_task_list=[
*threat_decay_task_list, *sensitivity_task_list,
*access_task_list],
dependent_task_list=individual_degradation_task_list,
task_name=f'tot_degradation_{row["decay"]}{lulc_key}_{threat}')
# Compute habitat quality
@ -846,53 +856,56 @@ def _calculate_habitat_quality(deg_hab_raster_list, quality_out_path, ksq):
target_path=quality_out_path)
def _calculate_total_degradation(
deg_raster_list, deg_sum_raster_path, weight_list):
"""Calculate habitat degradation.
def _calculate_individual_degradation(
filtered_threat_raster_path, sens_raster_path, weight_avg, access_raster_path,
indiv_threat_raster_path):
"""Calculate habitat degradation for a given threat.
Args:
deg_raster_list (list): list of string paths for the degraded
threat rasters.
deg_sum_raster_path (string): path to output the habitat quality
degradation raster.
weight_list (list): normalized weight for each threat corresponding
to threats in ``deg_raster_list``.
filtered_threat_raster_path (string): path for the filtered threat raster
sens_raster_path (string): path for the sensitivity raster
weight_avg (float): normalized weight for the threat
access_raster_path (string): path for the access raster
indiv_threat_raster_path (string): path to output the habitat quality
degradation raster for the individual threat.
Returns:
None
"""
def total_degradation(*arrays):
"""Computes the total degradation value.
def degradation(filtered_threat_array, sens_array, access_array):
"""Computes the degradation value for a given threat.
Args:
*raster (list): a list of numpy arrays of float type depicting
the adjusted threat value per pixel based on distance and
sensitivity. The values are in pairs so that the values for
each threat can be tracked:
[filtered_val_threat1, sens_val_threat1,
filtered_val_threat2, sens_val_threat2, ...]
There is an optional last value in the list which is the
access_raster value, but it is only present if
access_raster is not None.
filtered_threat_array (numpy.ndarray): filtered threat raster values
sens_array (numpy.ndarray): sensitivity raster values
access_array (numpy.ndarray): access raster based on values from the
shapefile if provided or a raster filled with all 1s if not
Returns:
The total degradation score for the pixel.
The degradation score for the pixel for the individual threat.
"""
# we can not be certain how many threats the user will enter,
# so we handle each filtered threat and sensitivity raster
# in pairs
sum_degradation = numpy.zeros(arrays[0].shape)
for index in range(len(arrays) // 2):
step = index * 2
sum_degradation += (
arrays[step] * arrays[step + 1] * weight_list[index])
# the last element in arrays is access
return sum_degradation * arrays[-1]
return filtered_threat_array * sens_array * weight_avg * access_array
pygeoprocessing.raster_map(
op=total_degradation,
rasters=deg_raster_list,
op=degradation,
rasters=[filtered_threat_raster_path, sens_raster_path, access_raster_path],
target_path=indiv_threat_raster_path)
def _calculate_total_degradation(indiv_deg_raster_list, deg_sum_raster_path):
"""Calculate total habitat degradation.
Args:
indiv_deg_raster_list (list): list of string paths for the degraded
threat rasters for each threat.
deg_sum_raster_path (string): path to output the habitat quality
degradation raster.
Returns:
None
"""
pygeoprocessing.raster_map(
op=lambda *indiv_arrays: numpy.sum(indiv_arrays, axis=0),
rasters=indiv_deg_raster_list,
target_path=deg_sum_raster_path)
@ -986,7 +999,7 @@ def _generate_rarity_csv(rarity_dict, target_csv_path):
lulc_codes = sorted(rarity_dict)
with open(target_csv_path, 'w', newline='') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['lulc_code', 'rarity_value'])
writer.writerow(['lucode', 'rarity_value'])
for lulc_code in lulc_codes:
writer.writerow([lulc_code, rarity_dict[lulc_code]])
@ -1055,7 +1068,7 @@ def _decay_distance(dist_raster_path, max_dist, decay_type, target_path):
dist_raster_path (string): a filepath for the raster to decay.
The raster is expected to be a euclidean distance transform with
values measuring distance in pixels.
max_dist (float): max distance of threat in KM.
max_dist (float): max distance of threat in meters.
decay_type (string): a string defining which decay method to use.
Options include: 'linear' | 'exponential'.
target_path (string): a filepath for a float output raster.
@ -1067,12 +1080,9 @@ def _decay_distance(dist_raster_path, max_dist, decay_type, target_path):
threat_pixel_size = pygeoprocessing.get_raster_info(
dist_raster_path)['pixel_size']
# convert max distance (given in KM) to meters
max_dist_m = max_dist * 1000
# convert max distance from meters to the number of pixels that
# represents on the raster
max_dist_pixel = max_dist_m / abs(threat_pixel_size[0])
max_dist_pixel = max_dist / abs(threat_pixel_size[0])
LOGGER.debug(f'Max distance in pixels: {max_dist_pixel}')
def linear_op(dist):
@ -1187,6 +1197,41 @@ def validate(args, limit_to=None):
invalid_keys.add('sensitivity_table_path')
# check that max_dist and weight values are included in the
# threats table and that max_dist >= 0
invalid_max_dist = []
missing_max_dist = []
missing_weight = []
for threat, row in threat_df.iterrows():
if row['max_dist'] == '':
missing_max_dist.append(threat)
elif row['max_dist'] <= 0:
invalid_max_dist.append(threat)
if row['weight'] == '':
missing_weight.append(threat)
if invalid_max_dist:
validation_warnings.append((
['threats_table_path'],
INVALID_MAX_DIST_MSG.format(threat_list=invalid_max_dist)
))
if missing_max_dist:
validation_warnings.append((
['threats_table_path'],
MISSING_MAX_DIST_MSG.format(threat_list=missing_max_dist)
))
if missing_weight:
validation_warnings.append((
['threats_table_path'],
MISSING_WEIGHT_MSG.format(threat_list=missing_weight)
))
if invalid_max_dist or missing_max_dist or missing_weight:
invalid_keys.add('threats_table_path')
# Validate threat raster paths and their nodata values
bad_threat_paths = []
duplicate_paths = []
@ -1233,8 +1278,6 @@ def validate(args, limit_to=None):
validation_warnings.append((
['threats_table_path'],
DUPLICATE_PATHS_MSG + str(duplicate_paths)))
if 'threats_table_path' not in invalid_keys:
invalid_keys.add('threats_table_path')
invalid_keys.add('threats_table_path')
return validation_warnings

1
src/natcap/invest/hra.py
View File

@ -50,6 +50,7 @@ _DEFAULT_GTIFF_CREATION_OPTIONS = (
'BLOCKXSIZE=256', 'BLOCKYSIZE=256')
MODEL_SPEC = {
"model_id": "habitat_risk_assessment",
"model_name": MODEL_METADATA["habitat_risk_assessment"].model_title,
"pyname": MODEL_METADATA["habitat_risk_assessment"].pyname,
"userguide": MODEL_METADATA["habitat_risk_assessment"].userguide,

79
src/natcap/invest/managed_raster/LRUCache.h Normal file
View File

@ -0,0 +1,79 @@
#ifndef __LRUCACHE_H_INCLUDED__
#define __LRUCACHE_H_INCLUDED__
#include <list>
#include <map>
#include <assert.h>
using namespace std;
template <class KEY_T, class VAL_T,
typename ListIter = typename list<pair<KEY_T,VAL_T>>::iterator,
typename MapIter = typename map<KEY_T, ListIter>::iterator> class LRUCache {
private:
// item_list keeps track of the order of which elements have been accessed
// element at begin is most recent, element at end is least recent.
// first element in the pair is its key while the second is the element
list<pair<KEY_T,VAL_T>> item_list;
// item_map maps an element's key to its location in the `item_list`
// used to make lookups O(log n) time
map<KEY_T, ListIter> item_map;
size_t cache_size;
void clean(list<pair<KEY_T, VAL_T>> &removed_value_list) {
while(item_map.size() > cache_size) {
ListIter last_it = item_list.end();
last_it--;
removed_value_list.push_back(
make_pair(last_it->first, last_it->second));
item_map.erase(last_it->first);
item_list.pop_back();
}
};
public:
LRUCache(int cache_size_):cache_size(cache_size_) {
;
};
ListIter begin() {
return item_list.begin();
}
ListIter end() {
return item_list.end();
}
// Insert a new key-value pair into the cache.
void put(
const KEY_T &key, const VAL_T &val,
list<pair<KEY_T, VAL_T>> &removed_value_list) {
MapIter it = item_map.find(key);
if(it != item_map.end()){
// it's already in the cache, delete the location in the item
// list and in the lookup map
item_list.erase(it->second);
item_map.erase(it);
}
// insert a new item in the front since it's most recently used
item_list.push_front(make_pair(key, val));
// record its iterator in the map
item_map.insert(make_pair(key, item_list.begin()));
// possibly remove any elements that have exceeded the cache size
return clean(removed_value_list);
};
// Return whether a key exists in the cache.
bool exist(const KEY_T &key) {
return (item_map.count(key) > 0);
};
// Return the cached value associated with a key.
VAL_T& get(const KEY_T &key) {
MapIter it = item_map.find(key);
assert(it != item_map.end());
// move the element to the front of the list
item_list.splice(item_list.begin(), item_list, it->second);
return it->second->second;
};
};
#endif

865
src/natcap/invest/managed_raster/ManagedRaster.h Normal file
View File

@ -0,0 +1,865 @@
#ifndef NATCAP_INVEST_MANAGEDRASTER_H_
#define NATCAP_INVEST_MANAGEDRASTER_H_
#include "gdal.h"
#include "gdal_priv.h"
#include <Python.h>
#include <iostream>
#include <string>
#include "LRUCache.h"
int MANAGED_RASTER_N_BLOCKS = static_cast<int>(pow(2, 6));
// given the pixel neighbor numbering system
// 3 2 1
// 4 x 0
// 5 6 7
// These offsets are for the neighbor rows and columns
int ROW_OFFSETS[8] = {0, -1, -1, -1, 0, 1, 1, 1};
int COL_OFFSETS[8] = {1, 1, 0, -1, -1, -1, 0, 1};
int FLOW_DIR_REVERSE_DIRECTION[8] = {4, 5, 6, 7, 0, 1, 2, 3};
// if a pixel `x` has a neighbor `n` in position `i`,
// then `n`'s neighbor in position `inflow_offsets[i]`
// is the original pixel `x`
int INFLOW_OFFSETS[8] = {4, 5, 6, 7, 0, 1, 2, 3};
typedef std::pair<int, double*> BlockBufferPair;
class D8 {};
class MFD {};
enum class LogLevel {debug, info, warning, error};
// Largely copied from:
// https://gist.github.com/hensing/0db3f8e3a99590006368
static void log_msg(LogLevel level, string msg)
{
static PyObject *logging = NULL;
static PyObject *pyString = NULL;
// import logging module on demand
if (logging == NULL) {
logging = PyImport_ImportModuleNoBlock("logging");
if (logging == NULL) {
PyErr_SetString(PyExc_ImportError,
"Could not import module 'logging'");
}
}
// build msg-string
pyString = Py_BuildValue("s", msg.c_str());
// call function depending on log level
switch (level)
{
case LogLevel::debug:
PyObject_CallMethod(logging, "debug", "O", pyString);
break;
case LogLevel::info:
PyObject_CallMethod(logging, "info", "O", pyString);
break;
case LogLevel::warning:
PyObject_CallMethod(logging, "warn", "O", pyString);
break;
case LogLevel::error:
PyObject_CallMethod(logging, "error", "O", pyString);
break;
}
Py_DECREF(pyString);
}
class NeighborTuple {
public:
int direction, x, y;
float flow_proportion;
NeighborTuple () {}
NeighborTuple (int direction, int x, int y, float flow_proportion) {
this->direction = direction;
this->x = x;
this->y = y;
this->flow_proportion = flow_proportion;
}
};
class ManagedRaster {
public:
LRUCache<int, double*>* lru_cache;
std::set<int> dirty_blocks;
int* actualBlockWidths;
int block_xsize;
int block_ysize;
int block_xmod;
int block_ymod;
int block_xbits;
int block_ybits;
long raster_x_size;
long raster_y_size;
int block_nx;
int block_ny;
char* raster_path;
int band_id;
GDALDataset* dataset;
GDALRasterBand* band;
int write_mode;
int closed;
double nodata;
double* geotransform;
int hasNodata;
ManagedRaster() { }
// Creates new instance of ManagedRaster. Opens the raster with GDAL,
// stores important information about the dataset, and creates a cache
// that will be used to efficiently read blocks from the raster.
// Args:
// raster_path: path to raster that has block sizes that are
// powers of 2. If not, an exception is raised.
// band_id: which band in `raster_path` to index. Uses GDAL
// notation that starts at 1.
// write_mode: if true, this raster is writable and dirty
// memory blocks will be written back to the raster as blocks
// are swapped out of the cache or when the object deconstructs.
ManagedRaster(char* raster_path, int band_id, bool write_mode)
: raster_path { raster_path }
, band_id { band_id }
, write_mode { write_mode }
{
GDALAllRegister();
dataset = (GDALDataset *) GDALOpen( raster_path, GA_Update );
raster_x_size = dataset->GetRasterXSize();
raster_y_size = dataset->GetRasterYSize();
if (band_id < 1 or band_id > dataset->GetRasterCount()) {
throw std::invalid_argument(
"Error: band ID is not a valid band number. "
"This error is happening in the ManagedRaster.h extension.");
}
band = dataset->GetRasterBand(band_id);
band->GetBlockSize( &block_xsize, &block_ysize );
block_xmod = block_xsize - 1;
block_ymod = block_ysize - 1;
nodata = band->GetNoDataValue( &hasNodata );
geotransform = (double *) CPLMalloc(sizeof(double) * 6);
dataset->GetGeoTransform(geotransform);
if (((block_xsize & (block_xsize - 1)) != 0) or (
(block_ysize & (block_ysize - 1)) != 0)) {
throw std::invalid_argument(
"Error: Block size is not a power of two. "
"This error is happening in the ManagedRaster.h extension.");
}
block_xbits = static_cast<int>(log2(block_xsize));
block_ybits = static_cast<int>(log2(block_ysize));
// integer floor division
block_nx = (raster_x_size + block_xsize - 1) / block_xsize;
block_ny = (raster_y_size + block_ysize - 1) / block_ysize;
int actual_x = 0;
int actual_y = 0;
actualBlockWidths = (int *) CPLMalloc(sizeof(int) * block_nx * block_ny);
for (int block_yi = 0; block_yi < block_ny; block_yi++) {
for (int block_xi = 0; block_xi < block_nx; block_xi++) {
band->GetActualBlockSize(block_xi, block_yi, &actual_x, &actual_y);
actualBlockWidths[block_yi * block_nx + block_xi] = actual_x;
}
}
lru_cache = new LRUCache<int, double*>(MANAGED_RASTER_N_BLOCKS);
closed = 0;
}
// Sets the pixel at `xi,yi` to `value`
void inline set(long xi, long yi, double value) {
int block_xi = xi / block_xsize;
int block_yi = yi / block_ysize;
// this is the flat index for the block
int block_index = block_yi * block_nx + block_xi;
if (not lru_cache->exist(block_index)) {
_load_block(block_index);
}
int idx = ((yi & block_ymod) * actualBlockWidths[block_index]) + (xi & block_xmod);
lru_cache->get(block_index)[idx] = value;
if (write_mode) {
std::set<int>::iterator dirty_itr = dirty_blocks.find(block_index);
if (dirty_itr == dirty_blocks.end()) {
dirty_blocks.insert(block_index);
}
}
}
// Returns the value of the pixel at `xi,yi`.
double inline get(long xi, long yi) {
int block_xi = xi / block_xsize;
int block_yi = yi / block_ysize;
// this is the flat index for the block
int block_index = block_yi * block_nx + block_xi;
if (not lru_cache->exist(block_index)) {
_load_block(block_index);
}
double* block = lru_cache->get(block_index);
// Using the property n % 2^i = n & (2^i - 1)
// to efficienty compute the modulo: yi % block_xsize
int idx = ((yi & block_ymod) * actualBlockWidths[block_index]) + (xi & block_xmod);
double value = block[idx];
return value;
}
// Reads a block from the raster and saves it to the cache.
// Args:
// block_index: Index of the block to read, counted from the top-left
void _load_block(int block_index) {
int block_xi = block_index % block_nx;
int block_yi = block_index / block_nx;
// we need the offsets to subtract from global indexes for cached array
int xoff = block_xi << block_xbits;
int yoff = block_yi << block_ybits;
double *double_buffer;
list<BlockBufferPair> removed_value_list;
// determine the block aligned xoffset for read as array
// initially the win size is the same as the block size unless
// we're at the edge of a raster
int win_xsize = block_xsize;
int win_ysize = block_ysize;
// load a new block
if ((xoff + win_xsize) > raster_x_size) {
win_xsize = win_xsize - (xoff + win_xsize - raster_x_size);
}
if ((yoff + win_ysize) > raster_y_size) {
win_ysize = win_ysize - (yoff + win_ysize - raster_y_size);
}
double *pafScanline = (double *) CPLMalloc(sizeof(double) * win_xsize * win_ysize);
CPLErr err = band->RasterIO(GF_Read, xoff, yoff, win_xsize, win_ysize,
pafScanline, win_xsize, win_ysize, GDT_Float64,
0, 0 );
if (err != CE_None) {
std::cerr << "Error reading block\n";
}
lru_cache->put(block_index, pafScanline, removed_value_list);
while (not removed_value_list.empty()) {
// write the changed value back if desired
double_buffer = removed_value_list.front().second;
if (write_mode) {
block_index = removed_value_list.front().first;
// write back the block if it's dirty
std::set<int>::iterator dirty_itr = dirty_blocks.find(block_index);
if (dirty_itr != dirty_blocks.end()) {
dirty_blocks.erase(dirty_itr);
block_xi = block_index % block_nx;
block_yi = block_index / block_nx;
xoff = block_xi << block_xbits;
yoff = block_yi << block_ybits;
win_xsize = block_xsize;
win_ysize = block_ysize;
if (xoff + win_xsize > raster_x_size) {
win_xsize = win_xsize - (xoff + win_xsize - raster_x_size);
}
if (yoff + win_ysize > raster_y_size) {
win_ysize = win_ysize - (yoff + win_ysize - raster_y_size);
}
err = band->RasterIO( GF_Write, xoff, yoff, win_xsize, win_ysize,
double_buffer, win_xsize, win_ysize, GDT_Float64, 0, 0 );
if (err != CE_None) {
std::cerr << "Error writing block\n";
}
}
}
CPLFree(double_buffer);
removed_value_list.pop_front();
}
}
// Closes the ManagedRaster and frees up resources.
// This call writes any dirty blocks to disk, frees up the memory
// allocated as part of the cache, and frees all GDAL references.
// Any subsequent calls to any other functions in _ManagedRaster will
// have undefined behavior.
void close() {
if (closed) {
return;
}
closed = 1;
double *double_buffer;
int block_xi;
int block_yi;
int block_index;
// initially the win size is the same as the block size unless
// we're at the edge of a raster
int win_xsize;
int win_ysize;
// we need the offsets to subtract from global indexes for cached array
int xoff;
int yoff;
if (not write_mode) {
for (auto it = lru_cache->begin(); it != lru_cache->end(); it++) {
// write the changed value back if desired
CPLFree(it->second);
}
GDALClose( (GDALDatasetH) dataset );
return;
}
// if we get here, we're in write_mode
std::set<int>::iterator dirty_itr;
for (auto it = lru_cache->begin(); it != lru_cache->end(); it++) {
double_buffer = it->second;
block_index = it->first;
// write to disk if block is dirty
dirty_itr = dirty_blocks.find(block_index);
if (dirty_itr != dirty_blocks.end()) {
dirty_blocks.erase(dirty_itr);
block_xi = block_index % block_nx;
block_yi = block_index / block_nx;
// we need the offsets to subtract from global indexes for
// cached array
xoff = block_xi << block_xbits;
yoff = block_yi << block_ybits;
win_xsize = block_xsize;
win_ysize = block_ysize;
// clip window sizes if necessary
if (xoff + win_xsize > raster_x_size) {
win_xsize = win_xsize - (xoff + win_xsize - raster_x_size);
}
if (yoff + win_ysize > raster_y_size) {
win_ysize = win_ysize - (yoff + win_ysize - raster_y_size);
}
CPLErr err = band->RasterIO( GF_Write, xoff, yoff, win_xsize, win_ysize,
double_buffer, win_xsize, win_ysize, GDT_Float64, 0, 0 );
if (err != CE_None) {
std::cerr << "Error writing block\n";
}
}
CPLFree(double_buffer);
}
GDALClose( (GDALDatasetH) dataset );
delete lru_cache;
free(actualBlockWidths);
}
};
// Represents a flow direction raster, which may be of type MFD or D8
template<class T>
class ManagedFlowDirRaster: public ManagedRaster {
public:
ManagedFlowDirRaster() {}
ManagedFlowDirRaster(char* raster_path, int band_id, bool write_mode)
: ManagedRaster(raster_path, band_id, write_mode) {}
// Checks if a given pixel is a local high point. (MFD implementation)
// Args:
// xi: x coord in pixel space of the pixel to consider
// yi: y coord in pixel space of the pixel to consider
// Returns:
// true if the pixel is a local high point, i.e. it has no
// upslope neighbors; false otherwise.
template<typename T_ = T, std::enable_if_t<std::is_same<T_, MFD>::value>* = nullptr>
bool is_local_high_point(int xi, int yi) {
int flow_dir_j, flow_ji;
long xj, yj;
for (int n_dir = 0; n_dir < 8; n_dir++) {
xj = xi + COL_OFFSETS[n_dir];
yj = yi + ROW_OFFSETS[n_dir];
if (xj < 0 or xj >= raster_x_size or yj < 0 or yj >= raster_y_size) {
continue;
}
flow_dir_j = static_cast<int>(get(xj, yj));
flow_ji = (0xF & (flow_dir_j >> (4 * FLOW_DIR_REVERSE_DIRECTION[n_dir])));
if (flow_ji) {
return false;
}
}
return true;
}
// Checks if a given pixel is a local high point. (D8 implementation)
// Args:
// xi: x coord in pixel space of the pixel to consider
// yi: y coord in pixel space of the pixel to consider
// Returns:
// true if the pixel is a local high point, i.e. it has no
// upslope neighbors; false otherwise.
template<typename T_ = T, std::enable_if_t<std::is_same<T_, D8>::value>* = nullptr>
bool is_local_high_point(int xi, int yi) {
int flow_dir_j;
long xj, yj;
for (int n_dir = 0; n_dir < 8; n_dir++) {
xj = xi + COL_OFFSETS[n_dir];
yj = yi + ROW_OFFSETS[n_dir];
if (xj < 0 or xj >= raster_x_size or yj < 0 or yj >= raster_y_size) {
continue;
}
flow_dir_j = static_cast<int>(get(xj, yj));
if (flow_dir_j == FLOW_DIR_REVERSE_DIRECTION[n_dir]) {
return false;
}
}
return true;
}
};
// Represents a pixel in a ManagedFlowDirectionRaster
template<class T>
class Pixel {
public:
ManagedFlowDirRaster<T> raster;
int x;
int y;
int val;
Pixel() {}
Pixel(ManagedFlowDirRaster<T> raster, int x, int y) : raster(raster), x(x), y(y) {
double v = raster.get(x, y);
val = static_cast<int>(v);
}
};
// Returned by the `.end()` method of Neighbor iterable classes
static inline NeighborTuple endVal = NeighborTuple(8, -1, -1, -1);
// Iterates over all eight neighboring pixels of a given pixel
// This and subsequent iterator classes were written with a lot of help from
// https://internalpointers.com/post/writing-custom-iterators-modern-cpp
template<class T>
class NeighborIterator {
public:
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = NeighborTuple;
using pointer = NeighborTuple*;
using reference = NeighborTuple&;
Pixel<T> pixel;
pointer m_ptr = nullptr;
int i = 0;
NeighborIterator() {}
NeighborIterator(NeighborTuple* n) { m_ptr = n; }
NeighborIterator(const Pixel<T> pixel) : pixel(pixel) { next(); }
reference operator*() const { return *m_ptr; }
pointer operator->() { return m_ptr; }
// Prefix increment
NeighborIterator<T>& operator++() { next(); return *this; }
// Postfix increment
NeighborIterator<T> operator++(int) { NeighborIterator<T> tmp = *this; ++(*this); return tmp; }
friend bool operator== (const NeighborIterator& a, const NeighborIterator& b) {
return a.m_ptr == b.m_ptr;
};
friend bool operator!= (const NeighborIterator& a, const NeighborIterator& b) {
return a.m_ptr != b.m_ptr;
};
// Increments the pointer to the next neighbor
virtual void next() {
long xj, yj, flow;
if (i == 8) {
m_ptr = &endVal;
return;
}
xj = pixel.x + COL_OFFSETS[i];
yj = pixel.y + ROW_OFFSETS[i];
flow = (pixel.val >> (i * 4)) & 0xF;
m_ptr = new NeighborTuple(i, xj, yj, static_cast<float>(flow));
i++;
}
};
// Iterates over neighbor pixels that are downslope of a given pixel,
// in either MFD or D8 mode
template<class T>
class DownslopeNeighborIterator: public NeighborIterator<T> {
public:
DownslopeNeighborIterator(): NeighborIterator<T>() {}
DownslopeNeighborIterator(NeighborTuple* n): NeighborIterator<T>(n) {}
DownslopeNeighborIterator(const Pixel<T> p) {
this->pixel = p;
next();
}
DownslopeNeighborIterator<T>& operator++() { next(); return *this; }
DownslopeNeighborIterator<T> operator++(int) { DownslopeNeighborIterator<T> tmp = *this; ++(*this); return tmp; }
// Increments the pointer to the next downslope neighbor (MFD)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, MFD>::value>* = nullptr>
void next() {
long xj, yj, flow;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->i++;
next();
return;
}
flow = (this->pixel.val >> (this->i * 4)) & 0xF;
if (flow) {
this->m_ptr = new NeighborTuple(this->i, xj, yj, static_cast<float>(flow));
this->i++;
return;
} else {
this->i++;
next();
}
}
// Increments the pointer to the next downslope neighbor (D8)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, D8>::value>* = nullptr>
void next() {
long xj, yj;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->pixel.val];
yj = this->pixel.y + ROW_OFFSETS[this->pixel.val];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->m_ptr = &endVal;
return;
}
this->i = 8;
this->m_ptr = new NeighborTuple(this->pixel.val, xj, yj, 1);
return;
}
};
// Iterates over neighbor pixels that are downslope of a given pixel,
// without skipping pixels that are out-of-bounds of the raster,
// in either MFD or D8 mode
template<class T>
class DownslopeNeighborNoSkipIterator: public NeighborIterator<T> {
public:
DownslopeNeighborNoSkipIterator(): NeighborIterator<T>() {}
DownslopeNeighborNoSkipIterator(NeighborTuple* n): NeighborIterator<T>(n) {}
DownslopeNeighborNoSkipIterator(const Pixel<T> p) {
this->pixel = p;
next();
}
DownslopeNeighborNoSkipIterator<T>& operator++() { next(); return *this; }
DownslopeNeighborNoSkipIterator<T> operator++(int) { DownslopeNeighborNoSkipIterator<T> tmp = *this; ++(*this); return tmp; }
// Increments the pointer to the next downslope neighbor (MFD)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, MFD>::value>* = nullptr>
void next() {
long xj, yj, flow;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
flow = (this->pixel.val >> (this->i * 4)) & 0xF;
if (flow) {
this->m_ptr = new NeighborTuple(this->i, xj, yj, static_cast<float>(flow));
this->i++;
return;
} else {
this->i++;
next();
}
}
// Increments the pointer to the next downslope neighbor (D8)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, D8>::value>* = nullptr>
void next() {
long xj, yj;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->pixel.val];
yj = this->pixel.y + ROW_OFFSETS[this->pixel.val];
this->i = 8;
this->m_ptr = new NeighborTuple(this->pixel.val, xj, yj, 1);
return;
}
};
// Iterates over neighbor pixels that are upslope of a given pixel,
// in either MFD or D8 mode
template<class T>
class UpslopeNeighborIterator: public NeighborIterator<T> {
public:
UpslopeNeighborIterator(): NeighborIterator<T>() {}
UpslopeNeighborIterator(NeighborTuple* n): NeighborIterator<T>(n) {}
UpslopeNeighborIterator(const Pixel<T> p) {
this->pixel = p;
next();
}
UpslopeNeighborIterator<T>& operator++() { next(); return *this; }
UpslopeNeighborIterator<T> operator++(int) { UpslopeNeighborIterator<T> tmp = *this; ++(*this); return tmp; }
// Increments the pointer to the next upslope neighbor (MFD)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, MFD>::value>* = nullptr>
void next() {
long xj, yj;
int flow_dir_j;
int flow_ji;
long flow_dir_j_sum;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->i++;
next();
return;
}
flow_dir_j = static_cast<int>(this->pixel.raster.get(xj, yj));
flow_ji = (0xF & (flow_dir_j >> (4 * FLOW_DIR_REVERSE_DIRECTION[this->i])));
if (flow_ji) {
flow_dir_j_sum = 0;
for (int idx = 0; idx < 8; idx++) {
flow_dir_j_sum += (flow_dir_j >> (idx * 4)) & 0xF;
}
this->m_ptr = new NeighborTuple(
this->i, xj, yj,
static_cast<float>(flow_ji) / static_cast<float>(flow_dir_j_sum));
this->i++;
return;
} else {
this->i++;
next();
}
}
// Increments the pointer to the next upslope neighbor (D8)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, D8>::value>* = nullptr>
void next() {
long xj, yj;
int flow_dir_j;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->i++;
next();
return;
}
flow_dir_j = static_cast<int>(this->pixel.raster.get(xj, yj));
if (flow_dir_j == FLOW_DIR_REVERSE_DIRECTION[this->i]) {
this->m_ptr = new NeighborTuple(this->i, xj, yj, 1);
this->i++;
return;
} else {
this->i++;
next();
}
}
};
// Iterates over neighbor pixels that are upslope of a given pixel,
// without dividing the flow_proportion, in either MFD or D8 mode
template<class T>
class UpslopeNeighborNoDivideIterator: public NeighborIterator<T> {
public:
UpslopeNeighborNoDivideIterator(): NeighborIterator<T>() {}
UpslopeNeighborNoDivideIterator(NeighborTuple* n): NeighborIterator<T>(n) {}
UpslopeNeighborNoDivideIterator(const Pixel<T> p) {
this->pixel = p;
next();
}
UpslopeNeighborNoDivideIterator<T>& operator++() { next(); return *this; }
UpslopeNeighborNoDivideIterator<T> operator++(int) { UpslopeNeighborNoDivideIterator<T> tmp = *this; ++(*this); return tmp; }
// Increments the pointer to the next upslope neighbor (MFD)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, MFD>::value>* = nullptr>
void next() {
long xj, yj;
int flow_dir_j;
int flow_ji;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->i++;
next();
return;
}
flow_dir_j = static_cast<int>(this->pixel.raster.get(xj, yj));
flow_ji = (0xF & (flow_dir_j >> (4 * FLOW_DIR_REVERSE_DIRECTION[this->i])));
if (flow_ji) {
this->m_ptr = new NeighborTuple(this->i, xj, yj, static_cast<float>(flow_ji));
this->i++;
return;
} else {
this->i++;
next();
}
}
// Increments the pointer to the next upslope neighbor (D8)
template<typename T_ = T, std::enable_if_t<std::is_same<T_, D8>::value>* = nullptr>
void next() {
long xj, yj;
int flow_dir_j;
delete this->m_ptr;
this->m_ptr = nullptr;
if (this->i == 8) {
this->m_ptr = &endVal;
return;
}
xj = this->pixel.x + COL_OFFSETS[this->i];
yj = this->pixel.y + ROW_OFFSETS[this->i];
if (xj < 0 or xj >= this->pixel.raster.raster_x_size or
yj < 0 or yj >= this->pixel.raster.raster_y_size) {
this->i++;
next();
return;
}
flow_dir_j = static_cast<int>(this->pixel.raster.get(xj, yj));
if (flow_dir_j == FLOW_DIR_REVERSE_DIRECTION[this->i]) {
this->m_ptr = new NeighborTuple(this->i, xj, yj, 1);
this->i++;
return;
} else {
this->i++;
next();
}
}
};
template<class T>
class Neighbors {
public:
Pixel<T> pixel;
Neighbors() {}
Neighbors(const Pixel<T> pixel): pixel(pixel) {}
NeighborIterator<T> begin() { return NeighborIterator<T>(pixel); }
NeighborIterator<T> end() { return NeighborIterator<T>(&endVal); }
};
template<class T>
class DownslopeNeighbors: public Neighbors<T> {
public:
using Neighbors<T>::Neighbors;
DownslopeNeighborIterator<T> begin() { return DownslopeNeighborIterator<T>(this->pixel); }
DownslopeNeighborIterator<T> end() { return DownslopeNeighborIterator<T>(&endVal); }
};
template<class T>
class DownslopeNeighborsNoSkip: public Neighbors<T> {
public:
using Neighbors<T>::Neighbors;
DownslopeNeighborNoSkipIterator<T> begin() { return DownslopeNeighborNoSkipIterator<T>(this->pixel); }
DownslopeNeighborNoSkipIterator<T> end() { return DownslopeNeighborNoSkipIterator<T>(&endVal); }
};
template<class T>
class UpslopeNeighbors: public Neighbors<T> {
public:
using Neighbors<T>::Neighbors;
UpslopeNeighborIterator<T> begin() { return UpslopeNeighborIterator<T>(this->pixel); }
UpslopeNeighborIterator<T> end() { return UpslopeNeighborIterator<T>(&endVal); }
};
template<class T>
class UpslopeNeighborsNoDivide: public Neighbors<T> {
public:
using Neighbors<T>::Neighbors;
UpslopeNeighborNoDivideIterator<T> begin() { return UpslopeNeighborNoDivideIterator<T>(this->pixel); }
UpslopeNeighborNoDivideIterator<T> end() { return UpslopeNeighborNoDivideIterator<T>(&endVal); }
};
// Note: I was concerned that checking each value for nan would be too slow, but
// I compared its performance against another implementation where we first reclassify
// nans to a regular float, and then skip the nan check, and that was much slower:
// https://github.com/natcap/invest/issues/1714#issuecomment-2762134419
inline bool is_close(double x, double y) {
if (isnan(x) and isnan(y)) {
return true;
}
return abs(x - y) <= (pow(10, -8) + pow(10, -05) * abs(y));
}
#endif // NATCAP_INVEST_MANAGEDRASTER_H_

0
src/natcap/invest/managed_raster/__init__.py Normal file
View File

136
src/natcap/invest/managed_raster/managed_raster.pxd Normal file
View File

@ -0,0 +1,136 @@
# cython: language_level=3
# distutils: language = c++
from libcpp.list cimport list as clist
from libcpp.pair cimport pair
from libcpp.set cimport set as cset
from libcpp.stack cimport stack
from libcpp.string cimport string
from libcpp.vector cimport vector
from libc.math cimport isnan
# this is a least recently used cache written in C++ in an external file,
# exposing here so ManagedRaster can use it
cdef extern from "LRUCache.h" nogil:
cdef cppclass LRUCache[KEY_T, VAL_T]:
LRUCache(int)
void put(KEY_T&, VAL_T&, clist[pair[KEY_T,VAL_T]]&)
clist[pair[KEY_T,VAL_T]].iterator begin()
clist[pair[KEY_T,VAL_T]].iterator end()
bint exist(KEY_T &)
VAL_T get(KEY_T &)
cdef extern from "ManagedRaster.h":
cdef cppclass ManagedRaster:
LRUCache[int, double*]* lru_cache
cset[int] dirty_blocks
int block_xsize
int block_ysize
int block_xmod
int block_ymod
int block_xbits
int block_ybits
long raster_x_size
long raster_y_size
int block_nx
int block_ny
int write_mode
string raster_path
int band_id
int closed
double nodata
ManagedRaster() except +
ManagedRaster(char*, int, bool) except +
void set(long xi, long yi, double value)
double get(long xi, long yi)
void _load_block(int block_index) except *
void close()
cdef cppclass ManagedFlowDirRaster[T]:
LRUCache[int, double*]* lru_cache
cset[int] dirty_blocks
int block_xsize
int block_ysize
int block_xmod
int block_ymod
int block_xbits
int block_ybits
long raster_x_size
long raster_y_size
int block_nx
int block_ny
int write_mode
string raster_path
int band_id
int closed
double nodata
bint is_local_high_point(int xi, int yi)
ManagedFlowDirRaster() except +
ManagedFlowDirRaster(char*, int, bool) except +
void set(long xi, long yi, double value)
double get(long xi, long yi)
void close()
cdef cppclass D8
cdef cppclass MFD
cdef cppclass NeighborTuple:
NeighborTuple() except +
NeighborTuple(int, int, int, float) except +
int direction, x, y
float flow_proportion
cdef cppclass DownslopeNeighborIterator[T]:
ManagedFlowDirRaster[T] raster
int col
int row
int n_dir
int flow_dir
int flow_dir_sum
DownslopeNeighborIterator()
DownslopeNeighborIterator(ManagedFlowDirRaster[T], int, int)
void next()
cdef cppclass DownslopeNeighborNoSkipIterator[T]:
ManagedFlowDirRaster[T] raster
int col
int row
int n_dir
int flow_dir
int flow_dir_sum
DownslopeNeighborNoSkipIterator()
DownslopeNeighborNoSkipIterator(ManagedFlowDirRaster[T], int, int)
void next()
cdef cppclass UpslopeNeighborIterator[T]:
ManagedFlowDirRaster[T] raster
int col
int row
int n_dir
int flow_dir
UpslopeNeighborIterator()
UpslopeNeighborIterator(ManagedFlowDirRaster[T], int, int)
void next()
cdef cppclass UpslopeNeighborNoDivideIterator[T]:
ManagedFlowDirRaster[T] raster
int col
int row
int n_dir
int flow_dir
UpslopeNeighborNoDivideIterator()
UpslopeNeighborNoDivideIterator(ManagedFlowDirRaster[T], int, int)
void next()
bint is_close(double, double)
int[8] INFLOW_OFFSETS
int[8] COL_OFFSETS
int[8] ROW_OFFSETS
int[8] FLOW_DIR_REVERSE_DIRECTION

73
src/natcap/invest/ndr/LRUCache.h
View File

@ -1,73 +0,0 @@
#ifndef __LRUCACHE_H_INCLUDED__
#define __LRUCACHE_H_INCLUDED__
#include <list>
#include <map>
#include <assert.h>
using namespace std;
template <class KEY_T, class VAL_T,
typename ListIter = typename list< pair<KEY_T,VAL_T> >::iterator,
typename MapIter = typename map<KEY_T, ListIter>::iterator > class LRUCache{
private:
// item_list keeps track of the order of which elements have been accessed
// element at begin is most recent, element at end is least recent.
// first element in the pair is its key while the second is the element
list< pair<KEY_T,VAL_T> > item_list;
// item_map maps an element's key to its location in the `item_list`
// used to make lookups O(log n) time
map<KEY_T, ListIter> item_map;
size_t cache_size;
private:
void clean(list< pair<KEY_T, VAL_T> > &removed_value_list){
while(item_map.size()>cache_size){
ListIter last_it = item_list.end(); last_it --;
removed_value_list.push_back(
make_pair(last_it->first, last_it->second));
item_map.erase(last_it->first);
item_list.pop_back();
}
};
public:
LRUCache(int cache_size_):cache_size(cache_size_){
;
};
ListIter begin() {
return item_list.begin();
}
ListIter end() {
return item_list.end();
}
void put(
const KEY_T &key, const VAL_T &val,
list< pair<KEY_T, VAL_T> > &removed_value_list) {
MapIter it = item_map.find(key);
if(it != item_map.end()){
// it's already in the cache, delete the location in the item
// list and in the lookup map
item_list.erase(it->second);
item_map.erase(it);
}
// insert a new item in the front since it's most recently used
item_list.push_front(make_pair(key,val));
// record its iterator in the map
item_map.insert(make_pair(key, item_list.begin()));
// possibly remove any elements that have exceeded the cache size
return clean(removed_value_list);
};
bool exist(const KEY_T &key){
return (item_map.count(key)>0);
};
VAL_T& get(const KEY_T &key){
MapIter it = item_map.find(key);
assert(it!=item_map.end());
// move the element to the front of the list
item_list.splice(item_list.begin(), item_list, it->second);
return it->second->second;
};
};
#endif

263
src/natcap/invest/ndr/effective_retention.h Normal file
View File

@ -0,0 +1,263 @@
#include "ManagedRaster.h"
#include <cmath>
#include <stack>
#include <ctime>
// Calculate flow downhill effective_retention to the channel.
// Args:
// flow_direction_path: a path to a flow direction raster (MFD or D8)
// stream_path: a path to a raster where 1 indicates a
// stream all other values ignored must be same dimensions and
// projection as flow_direction_path.
// retention_eff_lulc_path: a path to a raster indicating
// the maximum retention efficiency that the landcover on that
// pixel can accumulate.
// crit_len_path: a path to a raster indicating the critical
// length of the retention efficiency that the landcover on this
// pixel.
// effective_retention_path: path to a raster that is
// created by this call that contains a per-pixel effective
// sediment retention to the stream.
template<class T>
void run_effective_retention(
char* flow_direction_path,
char* stream_path,
char* retention_eff_lulc_path,
char* crit_len_path,
char* to_process_flow_directions_path,
char* effective_retention_path) {
// Within a stream, the effective retention is 0
int STREAM_EFFECTIVE_RETENTION = 0;
float effective_retention_nodata = -1;
stack<long> processing_stack;
ManagedFlowDirRaster flow_dir_raster = ManagedFlowDirRaster<T>(
flow_direction_path, 1, false);
ManagedRaster stream_raster = ManagedRaster(stream_path, 1, false);
ManagedRaster retention_eff_lulc_raster = ManagedRaster(
retention_eff_lulc_path, 1, false);
ManagedRaster crit_len_raster = ManagedRaster(crit_len_path, 1, false);
ManagedRaster to_process_flow_directions_raster = ManagedRaster(
to_process_flow_directions_path, 1, true);
ManagedRaster effective_retention_raster = ManagedRaster(
effective_retention_path, 1, true);
long n_cols = flow_dir_raster.raster_x_size;
long n_rows = flow_dir_raster.raster_y_size;
// cell sizes must be square, so no reason to test at this point.
double cell_size = stream_raster.geotransform[1];
double crit_len_nodata = crit_len_raster.nodata;
double retention_eff_nodata = retention_eff_lulc_raster.nodata;
long win_xsize, win_ysize, xoff, yoff;
long global_col, global_row;
unsigned long flat_index;
long flow_dir, neighbor_flow_dirs;
double current_step_factor, step_size, crit_len, retention_eff_lulc;
long neighbor_row, neighbor_col;
int neighbor_outflow_dir, neighbor_outflow_dir_mask, neighbor_process_flow_dir;
int outflow_dirs, dir_mask;
NeighborTuple neighbor;
bool should_seed;
double working_retention_eff;
DownslopeNeighborsNoSkip<T> dn_neighbors;
UpslopeNeighbors<T> up_neighbors;
bool has_outflow;
double neighbor_effective_retention;
double intermediate_retention;
string s;
long flow_dir_sum;
time_t last_log_time = time(NULL);
unsigned long n_pixels_processed = 0;
float total_n_pixels = flow_dir_raster.raster_x_size * flow_dir_raster.raster_y_size;
// efficient way to calculate ceiling division:
// a divided by b rounded up = (a + (b - 1)) / b
// note that / represents integer floor division
// https://stackoverflow.com/a/62032709/14451410
int n_col_blocks = (flow_dir_raster.raster_x_size + (flow_dir_raster.block_xsize - 1)) / flow_dir_raster.block_xsize;
int n_row_blocks = (flow_dir_raster.raster_y_size + (flow_dir_raster.block_ysize - 1)) / flow_dir_raster.block_ysize;
for (int row_block_index = 0; row_block_index < n_row_blocks; row_block_index++) {
yoff = row_block_index * flow_dir_raster.block_ysize;
win_ysize = flow_dir_raster.raster_y_size - yoff;
if (win_ysize > flow_dir_raster.block_ysize) {
win_ysize = flow_dir_raster.block_ysize;
}
for (int col_block_index = 0; col_block_index < n_col_blocks; col_block_index++) {
xoff = col_block_index * flow_dir_raster.block_xsize;
win_xsize = flow_dir_raster.raster_x_size - xoff;
if (win_xsize > flow_dir_raster.block_xsize) {
win_xsize = flow_dir_raster.block_xsize;
}
if (time(NULL) - last_log_time > 5) {
last_log_time = time(NULL);
log_msg(
LogLevel::info,
"Effective retention " + std::to_string(
100 * n_pixels_processed / total_n_pixels
) + " complete"
);
}
for (int row_index = 0; row_index < win_ysize; row_index++) {
global_row = yoff + row_index;
for (int col_index = 0; col_index < win_xsize; col_index++) {
global_col = xoff + col_index;
outflow_dirs = int(to_process_flow_directions_raster.get(
global_col, global_row));
should_seed = false;
// # see if this pixel drains to nodata or the edge, if so it's
// # a drain
for (int i = 0; i < 8; i++) {
dir_mask = 1 << i;
if ((outflow_dirs & dir_mask) > 0) {
neighbor_col = COL_OFFSETS[i] + global_col;
neighbor_row = ROW_OFFSETS[i] + global_row;
if (neighbor_col < 0 or neighbor_col >= n_cols or
neighbor_row < 0 or neighbor_row >= n_rows) {
should_seed = true;
outflow_dirs &= ~dir_mask;
} else {
// Only consider neighbor flow directions if the
// neighbor index is within the raster.
neighbor_flow_dirs = long(
to_process_flow_directions_raster.get(
neighbor_col, neighbor_row));
if (neighbor_flow_dirs == 0) {
should_seed = true;
outflow_dirs &= ~dir_mask;
}
}
}
}
if (should_seed) {
// mark all outflow directions processed
to_process_flow_directions_raster.set(
global_col, global_row, outflow_dirs);
processing_stack.push(global_row * n_cols + global_col);
}
}
}
while (processing_stack.size() > 0) {
// loop invariant, we don't push a cell on the stack that
// hasn't already been set for processing.
flat_index = processing_stack.top();
processing_stack.pop();
global_row = flat_index / n_cols; // integer floor division
global_col = flat_index % n_cols;
crit_len = crit_len_raster.get(global_col, global_row);
retention_eff_lulc = retention_eff_lulc_raster.get(global_col, global_row);
flow_dir = int(flow_dir_raster.get(global_col, global_row));
if (stream_raster.get(global_col, global_row) == 1) {
// if it's a stream, effective retention is 0.
effective_retention_raster.set(global_col, global_row, STREAM_EFFECTIVE_RETENTION);
} else if (is_close(crit_len, crit_len_nodata) or
is_close(retention_eff_lulc, retention_eff_nodata) or
flow_dir == 0) {
// if it's nodata, effective retention is nodata.
effective_retention_raster.set(
global_col, global_row, effective_retention_nodata);
} else {
working_retention_eff = 0;
dn_neighbors = DownslopeNeighborsNoSkip<T>(
Pixel<T>(flow_dir_raster, global_col, global_row));
has_outflow = false;
flow_dir_sum = 0;
for (auto neighbor: dn_neighbors) {
has_outflow = true;
flow_dir_sum += static_cast<long>(neighbor.flow_proportion);
if (neighbor.x < 0 or neighbor.x >= n_cols or
neighbor.y < 0 or neighbor.y >= n_rows) {
continue;
}
if (neighbor.direction % 2 == 1) {
step_size = cell_size * 1.41421356237;
} else {
step_size = cell_size;
}
// guard against a critical length factor that's 0
if (crit_len > 0) {
current_step_factor = exp(-5 * step_size / crit_len);
} else {
current_step_factor = 0;
}
neighbor_effective_retention = (
effective_retention_raster.get(
neighbor.x, neighbor.y));
// Case 1: downslope neighbor is a stream pixel
if (neighbor_effective_retention == STREAM_EFFECTIVE_RETENTION) {
intermediate_retention = (
retention_eff_lulc * (1 - current_step_factor));
// Case 2: the current LULC's retention exceeds the neighbor's retention.
} else if (retention_eff_lulc > neighbor_effective_retention) {
intermediate_retention = (
(neighbor_effective_retention * current_step_factor) +
(retention_eff_lulc * (1 - current_step_factor)));
// Case 3: the other 2 cases have not been hit.
} else {
intermediate_retention = neighbor_effective_retention;
}
working_retention_eff += (
intermediate_retention * neighbor.flow_proportion);
}
if (has_outflow) {
double v = working_retention_eff / flow_dir_sum;
effective_retention_raster.set(
global_col, global_row, v);
} else {
throw std::logic_error(
"got to a cell that has no outflow! This error is happening"
"in effective_retention.h");
}
}
// search upslope to see if we need to push a cell on the stack
// for i in range(8):
up_neighbors = UpslopeNeighbors<T>(Pixel<T>(flow_dir_raster, global_col, global_row));
for (auto neighbor: up_neighbors) {
neighbor_outflow_dir = INFLOW_OFFSETS[neighbor.direction];
neighbor_outflow_dir_mask = 1 << neighbor_outflow_dir;
neighbor_process_flow_dir = int(
to_process_flow_directions_raster.get(
neighbor.x, neighbor.y));
if (neighbor_process_flow_dir == 0) {
// skip, due to loop invariant this must be a nodata pixel
continue;
}
if ((neighbor_process_flow_dir & neighbor_outflow_dir_mask )== 0) {
// no outflow
continue;
}
// mask out the outflow dir that this iteration processed
neighbor_process_flow_dir &= ~neighbor_outflow_dir_mask;
to_process_flow_directions_raster.set(
neighbor.x, neighbor.y, neighbor_process_flow_dir);
if (neighbor_process_flow_dir == 0) {
// if 0 then all downslope have been processed,
// push on stack, otherwise another downslope pixel will
// pick it up
processing_stack.push(neighbor.y * n_cols + neighbor.x);
}
}
}
n_pixels_processed += win_xsize * win_ysize;
}
}
stream_raster.close();
crit_len_raster.close();
retention_eff_lulc_raster.close();
effective_retention_raster.close();
flow_dir_raster.close();
to_process_flow_directions_raster.close();
log_msg(LogLevel::info, "Effective retention 100% complete");
}

8
src/natcap/invest/ndr/effective_retention.pxd Normal file
View File

@ -0,0 +1,8 @@
cdef extern from "effective_retention.h":
void run_effective_retention[T](
char*,
char*,
char*,
char*,
char*,
char*) except +

282
src/natcap/invest/ndr/ndr.py
View File

@ -1,6 +1,5 @@
"""InVEST Nutrient Delivery Ratio (NDR) module."""
import copy
import itertools
import logging
import os
import pickle
@ -26,6 +25,7 @@ LOGGER = logging.getLogger(__name__)
MISSING_NUTRIENT_MSG = gettext('Either calc_n or calc_p must be True')
MODEL_SPEC = {
"model_id": "ndr",
"model_name": MODEL_METADATA["ndr"].model_title,
"pyname": MODEL_METADATA["ndr"].pyname,
"userguide": MODEL_METADATA["ndr"].userguide,
@ -94,16 +94,13 @@ MODEL_SPEC = {
"about": gettext(
"The distance after which it is assumed that this "
"LULC type retains the nutrient at its maximum "
"capacity. If nutrients travel a shorter distance "
"that this, the retention "
"efficiency will be less than the maximum value "
"eff_x, following an exponential decay.")},
"capacity.")},
"proportion_subsurface_n": {
"type": "ratio",
"required": "calc_n",
"about": gettext(
"The proportion of the total amount of nitrogen that "
"are dissolved into the subsurface. By default, this "
"is dissolved into the subsurface. By default, this "
"value should be set to 0, indicating that all "
"nutrients are delivered via surface flow. There is "
"no equivalent of this for phosphorus.")}
@ -143,6 +140,21 @@ MODEL_SPEC = {
"actually reaches the stream)."),
"name": gettext("Borselli k parameter"),
},
"runoff_proxy_av": {
"type": "number",
"units": u.none,
'expression': 'value > 0',
"required": False,
"name": gettext("average runoff proxy"),
"about": gettext(
"This parameter allows the user to specify a predefined "
"average value for the runoff proxy. This value is used "
"to normalize the Runoff Proxy raster when calculating "
"the Runoff Proxy Index (RPI). If a user does not specify "
"the runoff proxy average, this value will be automatically "
"calculated from the Runoff Proxy raster. The units will "
"be the same as those in the Runoff Proxy raster."),
},
"subsurface_critical_length_n": {
"type": "number",
"units": u.meter,
@ -162,7 +174,8 @@ MODEL_SPEC = {
"reached through subsurface flow. This characterizes the "
"retention due to biochemical degradation in soils. Required "
"if Calculate Nitrogen is selected.")
}
},
**spec_utils.FLOW_DIR_ALGORITHM
},
"outputs": {
"watershed_results_ndr.gpkg": {
@ -210,28 +223,28 @@ MODEL_SPEC = {
"about": "A pixel level map showing how much phosphorus from each pixel eventually reaches the stream by surface flow.",
"bands": {1: {
"type": "number",
"units": u.kilogram/u.pixel
"units": u.kilogram/u.hectare
}}
},
"n_surface_export.tif": {
"about": "A pixel level map showing how much nitrogen from each pixel eventually reaches the stream by surface flow.",
"bands": {1: {
"type": "number",
"units": u.kilogram/u.pixel
"units": u.kilogram/u.hectare
}}
},
"n_subsurface_export.tif": {
"about": "A pixel level map showing how much nitrogen from each pixel eventually reaches the stream by subsurface flow.",
"bands": {1: {
"type": "number",
"units": u.kilogram/u.pixel
"units": u.kilogram/u.hectare
}}
},
"n_total_export.tif": {
"about": "A pixel level map showing how much nitrogen from each pixel eventually reaches the stream by either flow.",
"bands": {1: {
"type": "number",
"units": u.kilogram/u.pixel
"units": u.kilogram/u.hectare
}}
},
"intermediate_outputs": {
@ -525,6 +538,9 @@ def execute(args):
args['k_param'] (number): The Borselli k parameter. This is a
calibration parameter that determines the shape of the
relationship between hydrologic connectivity.
args['runoff_proxy_av'] (number): (optional) The average runoff proxy.
Used to calculate the runoff proxy index. If not specified,
it will be automatically calculated.
args['subsurface_critical_length_n'] (number): The distance (traveled
subsurface and downslope) after which it is assumed that soil
retains nutrient at its maximum capacity, given in meters. If
@ -579,6 +595,11 @@ def execute(args):
args['biophysical_table_path'],
**MODEL_SPEC['args']['biophysical_table_path'])
# Ensure that if user doesn't explicitly assign a value,
# runoff_proxy_av = None
runoff_proxy_av = args.get("runoff_proxy_av")
runoff_proxy_av = float(runoff_proxy_av) if runoff_proxy_av else None
# these are used for aggregation in the last step
field_pickle_map = {}
@ -601,7 +622,10 @@ def execute(args):
base_raster_list, aligned_raster_list,
['near']*len(base_raster_list), dem_info['pixel_size'],
'intersection'),
kwargs={'base_vector_path_list': [args['watersheds_path']]},
kwargs={
'base_vector_path_list': [args['watersheds_path']],
'raster_align_index': 0 # align to the grid of the DEM
},
target_path_list=aligned_raster_list,
task_name='align rasters')
@ -668,35 +692,6 @@ def execute(args):
target_path_list=[f_reg['filled_dem_path']],
task_name='fill pits')
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(f_reg['filled_dem_path'], 1), f_reg['flow_direction_path']),
kwargs={'working_dir': intermediate_output_dir},
dependent_task_list=[fill_pits_task],
target_path_list=[f_reg['flow_direction_path']],
task_name='flow dir')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accum')
stream_extraction_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(f_reg['flow_accumulation_path'], 1),
(f_reg['flow_direction_path'], 1),
float(args['threshold_flow_accumulation']),
f_reg['stream_path']),
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream extraction')
calculate_slope_task = task_graph.add_task(
func=pygeoprocessing.calculate_slope,
args=((f_reg['filled_dem_path'], 1), f_reg['slope_path']),
@ -714,23 +709,90 @@ def execute(args):
dependent_task_list=[calculate_slope_task],
task_name='threshold slope')
if args['flow_dir_algorithm'] == 'MFD':
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(f_reg['filled_dem_path'], 1), f_reg['flow_direction_path']),
kwargs={'working_dir': intermediate_output_dir},
dependent_task_list=[fill_pits_task],
target_path_list=[f_reg['flow_direction_path']],
task_name='flow dir')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accum')
stream_extraction_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(f_reg['flow_accumulation_path'], 1),
(f_reg['flow_direction_path'], 1),
float(args['threshold_flow_accumulation']),
f_reg['stream_path']),
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream extraction')
s_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=((f_reg['flow_direction_path'], 1), f_reg['s_accumulation_path']),
kwargs={
'weight_raster_path_band': (f_reg['thresholded_slope_path'], 1)},
target_path_list=[f_reg['s_accumulation_path']],
dependent_task_list=[flow_dir_task, threshold_slope_task],
task_name='route s')
else: # D8
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_d8,
args=(
(f_reg['filled_dem_path'], 1), f_reg['flow_direction_path']),
kwargs={'working_dir': intermediate_output_dir},
dependent_task_list=[fill_pits_task],
target_path_list=[f_reg['flow_direction_path']],
task_name='flow dir')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_d8,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accum')
stream_extraction_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_d8,
kwargs=dict(
flow_accum_raster_path_band=(f_reg['flow_accumulation_path'], 1),
flow_threshold=float(args['threshold_flow_accumulation']),
target_stream_raster_path=f_reg['stream_path']),
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream extraction')
s_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_d8,
args=((f_reg['flow_direction_path'], 1), f_reg['s_accumulation_path']),
kwargs={
'weight_raster_path_band': (f_reg['thresholded_slope_path'], 1)},
target_path_list=[f_reg['s_accumulation_path']],
dependent_task_list=[flow_dir_task, threshold_slope_task],
task_name='route s')
runoff_proxy_index_task = task_graph.add_task(
func=_normalize_raster,
args=((f_reg['masked_runoff_proxy_path'], 1),
f_reg['runoff_proxy_index_path']),
kwargs={'user_provided_mean': runoff_proxy_av},
target_path_list=[f_reg['runoff_proxy_index_path']],
dependent_task_list=[align_raster_task, mask_runoff_proxy_task],
task_name='runoff proxy mean')
s_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=((f_reg['flow_direction_path'], 1), f_reg['s_accumulation_path']),
kwargs={
'weight_raster_path_band': (f_reg['thresholded_slope_path'], 1)},
target_path_list=[f_reg['s_accumulation_path']],
dependent_task_list=[flow_dir_task, threshold_slope_task],
task_name='route s')
s_bar_task = task_graph.add_task(
func=pygeoprocessing.raster_map,
kwargs=dict(
@ -762,27 +824,50 @@ def execute(args):
dependent_task_list=[threshold_slope_task],
task_name='s inv')
d_dn_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_mfd,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['d_dn_path']),
kwargs={'weight_raster_path_band': (
f_reg['s_factor_inverse_path'], 1)},
dependent_task_list=[stream_extraction_task, s_inv_task],
target_path_list=[f_reg['d_dn_path']],
task_name='d dn')
if args['flow_dir_algorithm'] == 'MFD':
d_dn_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_mfd,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['d_dn_path']),
kwargs={'weight_raster_path_band': (
f_reg['s_factor_inverse_path'], 1)},
dependent_task_list=[stream_extraction_task, s_inv_task],
target_path_list=[f_reg['d_dn_path']],
task_name='d dn')
dist_to_channel_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_mfd,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['dist_to_channel_path']),
dependent_task_list=[stream_extraction_task],
target_path_list=[f_reg['dist_to_channel_path']],
task_name='dist to channel')
dist_to_channel_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_mfd,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['dist_to_channel_path']),
dependent_task_list=[stream_extraction_task],
target_path_list=[f_reg['dist_to_channel_path']],
task_name='dist to channel')
else: # D8
d_dn_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_d8,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['d_dn_path']),
kwargs={'weight_raster_path_band': (
f_reg['s_factor_inverse_path'], 1)},
dependent_task_list=[stream_extraction_task, s_inv_task],
target_path_list=[f_reg['d_dn_path']],
task_name='d dn')
dist_to_channel_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_d8,
args=(
(f_reg['flow_direction_path'], 1),
(f_reg['stream_path'], 1),
f_reg['dist_to_channel_path']),
dependent_task_list=[stream_extraction_task],
target_path_list=[f_reg['dist_to_channel_path']],
task_name='dist to channel')
_ = task_graph.add_task(
func=sdr._calculate_what_drains_to_stream,
@ -869,7 +954,8 @@ def execute(args):
args=(
f_reg['flow_direction_path'],
f_reg['stream_path'], eff_path,
crit_len_path, effective_retention_path),
crit_len_path, effective_retention_path,
args['flow_dir_algorithm']),
target_path_list=[effective_retention_path],
dependent_task_list=[
stream_extraction_task, eff_task, crit_len_task],
@ -1029,12 +1115,15 @@ def execute(args):
# raster_map equation: Multiply a series of arrays element-wise
def _mult_op(*array_list): return numpy.prod(numpy.stack(array_list), axis=0)
# raster_map equation: Sum a list of arrays element-wise
def _sum_op(*array_list): return numpy.sum(array_list, axis=0)
# raster_map equation: calculate inverse of S factor
def _inverse_op(base_val): return numpy.where(base_val == 0, 0, 1 / base_val)
# raster_map equation: rescale and threshold slope between 0.005 and 1
def _slope_proportion_and_threshold_op(slope):
slope_fraction = slope / 100
@ -1135,8 +1224,13 @@ def _add_fields_to_shapefile(field_pickle_map, target_vector_path):
for feature in target_layer:
fid = feature.GetFID()
for field_name in field_pickle_map:
# Since pixel values are kg/(ha•yr), raster sum is (kg•px)/(ha•yr).
# To convert to kg/yr, multiply by ha/px.
pixel_area = field_summaries[field_name]['pixel_area']
ha_per_px = pixel_area / 10000
feature.SetField(
field_name, float(field_summaries[field_name][fid]['sum']))
field_name, float(
field_summaries[field_name][fid]['sum']) * ha_per_px)
# Save back to datasource
target_layer.SetFeature(feature)
target_layer = None
@ -1190,7 +1284,8 @@ def validate(args, limit_to=None):
return validation_warnings
def _normalize_raster(base_raster_path_band, target_normalized_raster_path):
def _normalize_raster(base_raster_path_band, target_normalized_raster_path,
user_provided_mean=None):
"""Calculate normalize raster by dividing by the mean value.
Args:
@ -1198,20 +1293,27 @@ def _normalize_raster(base_raster_path_band, target_normalized_raster_path):
mean.
target_normalized_raster_path (string): path to target normalized
raster from base_raster_path_band.
user_provided_mean (float, optional): user-provided average.
If provided, this value will be used instead of computing
the mean from the raster.
Returns:
None.
"""
value_sum, value_count = pygeoprocessing.raster_reduce(
function=lambda sum_count, block: # calculate both in one pass
(sum_count[0] + numpy.sum(block), sum_count[1] + block.size),
raster_path_band=base_raster_path_band,
initializer=(0, 0))
value_mean = value_sum
if value_count > 0:
value_mean /= value_count
if user_provided_mean is None:
value_sum, value_count = pygeoprocessing.raster_reduce(
function=lambda sum_count, block: ( # calculate both in one pass
sum_count[0] + numpy.sum(block), sum_count[1] + block.size),
raster_path_band=base_raster_path_band,
initializer=(0, 0))
value_mean = value_sum
if value_count > 0:
value_mean /= value_count
LOGGER.info(f"Normalizing raster ({base_raster_path_band[0]}) using "
f"auto-calculated mean: {value_mean}")
else:
value_mean = user_provided_mean
pygeoprocessing.raster_map(
op=lambda array: array if value_mean == 0 else array / value_mean,
@ -1221,29 +1323,25 @@ def _normalize_raster(base_raster_path_band, target_normalized_raster_path):
def _calculate_load(lulc_raster_path, lucode_to_load, target_load_raster):
"""Calculate load raster by mapping landcover and multiplying by area.
"""Calculate load raster by mapping landcover.
Args:
lulc_raster_path (string): path to integer landcover raster.
lucode_to_load (dict): a mapping of landcover IDs to per-area
nutrient load.
target_load_raster (string): path to target raster that will have
total load per pixel.
load values (kg/ha) mapped to pixels based on LULC.
Returns:
None.
"""
cell_area_ha = abs(numpy.prod(pygeoprocessing.get_raster_info(
lulc_raster_path)['pixel_size'])) * 0.0001
def _map_load_op(lucode_array):
"""Convert unit load to total load & handle nodata."""
result = numpy.empty(lucode_array.shape)
for lucode in numpy.unique(lucode_array):
try:
result[lucode_array == lucode] = (
lucode_to_load[lucode] * cell_area_ha)
result[lucode_array == lucode] = (lucode_to_load[lucode])
except KeyError:
raise KeyError(
'lucode: %d is present in the landuse raster but '
@ -1439,6 +1537,12 @@ def _aggregate_and_pickle_total(
base_raster_path_band, aggregate_vector_path,
working_dir=os.path.dirname(target_pickle_path))
# Write pixel area to pickle file so that _add_fields_to_shapefile
# can adjust totals as needed.
raster_info = pygeoprocessing.get_raster_info(base_raster_path_band[0])
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
result['pixel_area'] = pixel_area
with open(target_pickle_path, 'wb') as target_pickle_file:
pickle.dump(result, target_pickle_file)

568
src/natcap/invest/ndr/ndr_core.pyx
View File

@ -1,29 +1,16 @@
import tempfile
import logging
import os
import collections
import numpy
import pygeoprocessing
cimport numpy
cimport cython
from osgeo import gdal
from cython.operator cimport dereference as deref
from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cython.operator cimport dereference as deref
from cython.operator cimport preincrement as inc
from libcpp.pair cimport pair
from libcpp.set cimport set as cset
from libcpp.list cimport list as clist
from libcpp.stack cimport stack
from libcpp.map cimport map
from libc.math cimport atan
from libc.math cimport atan2
from libc.math cimport tan
from libc.math cimport sqrt
from libc.math cimport ceil
from libc.math cimport exp
from ..managed_raster.managed_raster cimport D8
from ..managed_raster.managed_raster cimport MFD
from .effective_retention cimport run_effective_retention
cdef extern from "time.h" nogil:
ctypedef int time_t
@ -31,336 +18,20 @@ cdef extern from "time.h" nogil:
LOGGER = logging.getLogger(__name__)
cdef double PI = 3.141592653589793238462643383279502884
# This module creates rasters with a memory xy block size of 2**BLOCK_BITS
cdef int BLOCK_BITS = 8
# Number of raster blocks to hold in memory at once per Managed Raster
cdef int MANAGED_RASTER_N_BLOCKS = 2**6
# Within a stream, the effective retention is 0
cdef int STREAM_EFFECTIVE_RETENTION = 0
cdef int is_close(double x, double y):
return abs(x-y) <= (1e-8+1e-05*abs(y))
# this is a least recently used cache written in C++ in an external file,
# exposing here so _ManagedRaster can use it
cdef extern from "LRUCache.h" nogil:
cdef cppclass LRUCache[KEY_T, VAL_T]:
LRUCache(int)
void put(KEY_T&, VAL_T&, clist[pair[KEY_T,VAL_T]]&)
clist[pair[KEY_T,VAL_T]].iterator begin()
clist[pair[KEY_T,VAL_T]].iterator end()
bint exist(KEY_T &)
VAL_T get(KEY_T &)
# this ctype is used to store the block ID and the block buffer as one object
# inside Managed Raster
ctypedef pair[int, double*] BlockBufferPair
# a class to allow fast random per-pixel access to a raster for both setting
# and reading pixels. Copied from src/pygeoprocessing/routing/routing.pyx,
# revision 891288683889237cfd3a3d0a1f09483c23489fca.
cdef class _ManagedRaster:
cdef LRUCache[int, double*]* lru_cache
cdef cset[int] dirty_blocks
cdef int block_xsize
cdef int block_ysize
cdef int block_xmod
cdef int block_ymod
cdef int block_xbits
cdef int block_ybits
cdef long raster_x_size
cdef long raster_y_size
cdef int block_nx
cdef int block_ny
cdef int write_mode
cdef bytes raster_path
cdef int band_id
cdef int closed
def __cinit__(self, raster_path, band_id, write_mode):
"""Create new instance of Managed Raster.
Args:
raster_path (char*): path to raster that has block sizes that are
powers of 2. If not, an exception is raised.
band_id (int): which band in `raster_path` to index. Uses GDAL
notation that starts at 1.
write_mode (boolean): if true, this raster is writable and dirty
memory blocks will be written back to the raster as blocks
are swapped out of the cache or when the object deconstructs.
Returns:
None.
"""
raster_info = pygeoprocessing.get_raster_info(raster_path)
self.raster_x_size, self.raster_y_size = raster_info['raster_size']
self.block_xsize, self.block_ysize = raster_info['block_size']
self.block_xmod = self.block_xsize-1
self.block_ymod = self.block_ysize-1
if not (1 <= band_id <= raster_info['n_bands']):
err_msg = (
"Error: band ID (%s) is not a valid band number. "
"This exception is happening in Cython, so it will cause a "
"hard seg-fault, but it's otherwise meant to be a "
"ValueError." % (band_id))
print(err_msg)
raise ValueError(err_msg)
self.band_id = band_id
if (self.block_xsize & (self.block_xsize - 1) != 0) or (
self.block_ysize & (self.block_ysize - 1) != 0):
# If inputs are not a power of two, this will at least print
# an error message. Unfortunately with Cython, the exception will
# present itself as a hard seg-fault, but I'm leaving the
# ValueError in here at least for readability.
err_msg = (
"Error: Block size is not a power of two: "
"block_xsize: %d, %d, %s. This exception is happening"
"in Cython, so it will cause a hard seg-fault, but it's"
"otherwise meant to be a ValueError." % (
self.block_xsize, self.block_ysize, raster_path))
print(err_msg)
raise ValueError(err_msg)
self.block_xbits = numpy.log2(self.block_xsize)
self.block_ybits = numpy.log2(self.block_ysize)
self.block_nx = (
self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize
self.block_ny = (
self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize
self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS)
self.raster_path = <bytes> raster_path
self.write_mode = write_mode
self.closed = 0
def __dealloc__(self):
"""Deallocate _ManagedRaster.
This operation manually frees memory from the LRUCache and writes any
dirty memory blocks back to the raster if `self.write_mode` is True.
"""
self.close()
def close(self):
"""Close the _ManagedRaster and free up resources.
This call writes any dirty blocks to disk, frees up the memory
allocated as part of the cache, and frees all GDAL references.
Any subsequent calls to any other functions in _ManagedRaster will
have undefined behavior.
"""
if self.closed:
return
self.closed = 1
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array = numpy.empty(
(self.block_ysize, self.block_xsize))
cdef double *double_buffer
cdef int block_xi
cdef int block_yi
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize
cdef int win_ysize
# we need the offsets to subtract from global indexes for cached array
cdef int xoff
cdef int yoff
cdef clist[BlockBufferPair].iterator it = self.lru_cache.begin()
cdef clist[BlockBufferPair].iterator end = self.lru_cache.end()
if not self.write_mode:
while it != end:
# write the changed value back if desired
PyMem_Free(deref(it).second)
inc(it)
return
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
# if we get here, we're in write_mode
cdef cset[int].iterator dirty_itr
while it != end:
double_buffer = deref(it).second
block_index = deref(it).first
# write to disk if block is dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for
# cached array
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
# clip window sizes if necessary
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = (
double_buffer[
(yi_copy << self.block_xbits) + xi_copy])
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
inc(it)
raster_band.FlushCache()
raster_band = None
raster = None
cdef inline void set(self, long xi, long yi, double value):
"""Set the pixel at `xi,yi` to `value`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))] = value
if self.write_mode:
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr == self.dirty_blocks.end():
self.dirty_blocks.insert(block_index)
cdef inline double get(self, long xi, long yi):
"""Return the value of the pixel at `xi,yi`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
return self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))]
cdef void _load_block(self, int block_index) except *:
cdef int block_xi = block_index % self.block_nx
cdef int block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for cached array
cdef int xoff = block_xi << self.block_xbits
cdef int yoff = block_yi << self.block_ybits
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array
cdef double *double_buffer
cdef clist[BlockBufferPair] removed_value_list
# determine the block aligned xoffset for read as array
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize = self.block_xsize
cdef int win_ysize = self.block_ysize
# load a new block
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (yoff+win_ysize - self.raster_y_size)
raster = gdal.OpenEx(self.raster_path, gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = raster_band.ReadAsArray(
xoff=xoff, yoff=yoff, win_xsize=win_xsize,
win_ysize=win_ysize).astype(
numpy.float64)
raster_band = None
raster = None
double_buffer = <double*>PyMem_Malloc(
(sizeof(double) << self.block_xbits) * win_ysize)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
double_buffer[(yi_copy<<self.block_xbits)+xi_copy] = (
block_array[yi_copy, xi_copy])
self.lru_cache.put(
<int>block_index, <double*>double_buffer, removed_value_list)
if self.write_mode:
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = numpy.empty(
(self.block_ysize, self.block_xsize), dtype=numpy.double)
while not removed_value_list.empty():
# write the changed value back if desired
double_buffer = removed_value_list.front().second
if self.write_mode:
block_index = removed_value_list.front().first
# write back the block if it's dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = double_buffer[
(yi_copy << self.block_xbits) + xi_copy]
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
removed_value_list.pop_front()
if self.write_mode:
raster_band = None
raster = None
def ndr_eff_calculation(
mfd_flow_direction_path, stream_path, retention_eff_lulc_path,
crit_len_path, effective_retention_path):
flow_direction_path, stream_path, retention_eff_lulc_path,
crit_len_path, effective_retention_path, algorithm):
"""Calculate flow downhill effective_retention to the channel.
Args:
mfd_flow_direction_path (string): a path to a raster with
pygeoprocessing.routing MFD flow direction values.
flow_direction_path (string): a path to a raster with
pygeoprocessing.routing flow direction values (MFD or D8).
stream_path (string): a path to a raster where 1 indicates a
stream all other values ignored must be same dimensions and
projection as mfd_flow_direction_path.
projection as flow_direction_path.
retention_eff_lulc_path (string): a path to a raster indicating
the maximum retention efficiency that the landcover on that
pixel can accumulate.
@ -377,40 +48,13 @@ def ndr_eff_calculation(
"""
cdef float effective_retention_nodata = -1.0
pygeoprocessing.new_raster_from_base(
mfd_flow_direction_path, effective_retention_path, gdal.GDT_Float32,
flow_direction_path, effective_retention_path, gdal.GDT_Float32,
[effective_retention_nodata])
fp, to_process_flow_directions_path = tempfile.mkstemp(
suffix='.tif', prefix='flow_to_process',
dir=os.path.dirname(effective_retention_path))
os.close(fp)
cdef int *row_offsets = [0, -1, -1, -1, 0, 1, 1, 1]
cdef int *col_offsets = [1, 1, 0, -1, -1, -1, 0, 1]
cdef int *inflow_offsets = [4, 5, 6, 7, 0, 1, 2, 3]
cdef long n_cols, n_rows
flow_dir_info = pygeoprocessing.get_raster_info(mfd_flow_direction_path)
n_cols, n_rows = flow_dir_info['raster_size']
cdef stack[long] processing_stack
stream_info = pygeoprocessing.get_raster_info(stream_path)
# cell sizes must be square, so no reason to test at this point.
cdef float cell_size = abs(stream_info['pixel_size'][0])
cdef _ManagedRaster stream_raster = _ManagedRaster(stream_path, 1, False)
cdef _ManagedRaster crit_len_raster = _ManagedRaster(
crit_len_path, 1, False)
cdef float crit_len_nodata = pygeoprocessing.get_raster_info(
crit_len_path)['nodata'][0]
cdef _ManagedRaster retention_eff_lulc_raster = _ManagedRaster(
retention_eff_lulc_path, 1, False)
cdef float retention_eff_nodata = pygeoprocessing.get_raster_info(
retention_eff_lulc_path)['nodata'][0]
cdef _ManagedRaster effective_retention_raster = _ManagedRaster(
effective_retention_path, 1, True)
cdef _ManagedRaster mfd_flow_direction_raster = _ManagedRaster(
mfd_flow_direction_path, 1, False)
# create direction raster in bytes
def _mfd_to_flow_dir_op(mfd_array):
result = numpy.zeros(mfd_array.shape, dtype=numpy.uint8)
@ -418,170 +62,36 @@ def ndr_eff_calculation(
result[:] |= ((((mfd_array >> (i*4)) & 0xF) > 0) << i).astype(numpy.uint8)
return result
# create direction raster in bytes
def _d8_to_flow_dir_op(d8_array):
result = numpy.zeros(d8_array.shape, dtype=numpy.uint8)
for i in range(8):
result[d8_array == i] = 1 << i
return result
flow_dir_op = _mfd_to_flow_dir_op if algorithm == 'MFD' else _d8_to_flow_dir_op
# convert mfd raster to binary mfd
# each value is an 8-digit binary number
# where 1 indicates that the pixel drains in that direction
# and 0 indicates that it does not drain in that direction
pygeoprocessing.raster_calculator(
[(mfd_flow_direction_path, 1)], _mfd_to_flow_dir_op,
[(flow_direction_path, 1)], flow_dir_op,
to_process_flow_directions_path, gdal.GDT_Byte, None)
cdef _ManagedRaster to_process_flow_directions_raster = _ManagedRaster(
to_process_flow_directions_path, 1, True)
cdef long col_index, row_index, win_xsize, win_ysize, xoff, yoff
cdef long global_col, global_row
cdef unsigned long flat_index
cdef long outflow_weight, outflow_weight_sum, flow_dir
cdef long ds_col, ds_row, i
cdef float current_step_factor, step_size, crit_len
cdef long neighbor_row, neighbor_col
cdef int neighbor_outflow_dir, neighbor_outflow_dir_mask, neighbor_process_flow_dir
cdef int outflow_dirs, dir_mask
for offset_dict in pygeoprocessing.iterblocks(
(mfd_flow_direction_path, 1), offset_only=True, largest_block=0):
win_xsize = offset_dict['win_xsize']
win_ysize = offset_dict['win_ysize']
xoff = offset_dict['xoff']
yoff = offset_dict['yoff']
for row_index in range(win_ysize):
global_row = yoff + row_index
for col_index in range(win_xsize):
global_col = xoff + col_index
outflow_dirs = <int>to_process_flow_directions_raster.get(
global_col, global_row)
should_seed = 0
# see if this pixel drains to nodata or the edge, if so it's
# a drain
for i in range(8):
dir_mask = 1 << i
if outflow_dirs & dir_mask > 0:
neighbor_col = col_offsets[i] + global_col
if neighbor_col < 0 or neighbor_col >= n_cols:
should_seed = 1
outflow_dirs &= ~dir_mask
neighbor_row = row_offsets[i] + global_row
if neighbor_row < 0 or neighbor_row >= n_rows:
should_seed = 1
outflow_dirs &= ~dir_mask
# Only consider neighbor flow directions if the
# neighbor index is within the raster.
if (neighbor_col >= 0
and neighbor_row >= 0
and neighbor_col < n_cols
and neighbor_row < n_rows):
neighbor_flow_dirs = (
to_process_flow_directions_raster.get(
neighbor_col, neighbor_row))
if neighbor_flow_dirs == 0:
should_seed = 1
outflow_dirs &= ~dir_mask
if should_seed:
# mark all outflow directions processed
to_process_flow_directions_raster.set(
global_col, global_row, outflow_dirs)
processing_stack.push(global_row*n_cols+global_col)
while processing_stack.size() > 0:
# loop invariant, we don't push a cell on the stack that
# hasn't already been set for processing.
flat_index = processing_stack.top()
processing_stack.pop()
global_row = flat_index // n_cols
global_col = flat_index % n_cols
crit_len = <float>crit_len_raster.get(global_col, global_row)
retention_eff_lulc = retention_eff_lulc_raster.get(
global_col, global_row)
flow_dir = <int>mfd_flow_direction_raster.get(
global_col, global_row)
if stream_raster.get(global_col, global_row) == 1:
# if it's a stream effective retention is 0.
effective_retention_raster.set(global_col, global_row, STREAM_EFFECTIVE_RETENTION)
elif (is_close(crit_len, crit_len_nodata) or
is_close(retention_eff_lulc, retention_eff_nodata) or
flow_dir == 0):
# if it's nodata, effective retention is nodata.
effective_retention_raster.set(
global_col, global_row, effective_retention_nodata)
else:
working_retention_eff = 0.0
outflow_weight_sum = 0
for i in range(8):
outflow_weight = (flow_dir >> (i*4)) & 0xF
if outflow_weight == 0:
continue
outflow_weight_sum += outflow_weight
ds_col = col_offsets[i] + global_col
if ds_col < 0 or ds_col >= n_cols:
continue
ds_row = row_offsets[i] + global_row
if ds_row < 0 or ds_row >= n_rows:
continue
if i % 2 == 1:
step_size = <float>(cell_size*1.41421356237)
else:
step_size = cell_size
# guard against a critical length factor that's 0
if crit_len > 0:
current_step_factor = <float>(
exp(-5*step_size/crit_len))
else:
current_step_factor = 0.0
neighbor_effective_retention = (
effective_retention_raster.get(ds_col, ds_row))
# Case 1: downslope neighbor is a stream pixel
if neighbor_effective_retention == STREAM_EFFECTIVE_RETENTION:
intermediate_retention = (
retention_eff_lulc * ( 1 - current_step_factor))
# Case 2: the current LULC's retention exceeds the neighbor's retention.
elif retention_eff_lulc > neighbor_effective_retention:
intermediate_retention = (
(neighbor_effective_retention * current_step_factor) +
(retention_eff_lulc * (1 - current_step_factor)))
# Case 3: the other 2 cases have not been hit.
else:
intermediate_retention = neighbor_effective_retention
working_retention_eff += intermediate_retention * outflow_weight
if outflow_weight_sum > 0:
working_retention_eff /= float(outflow_weight_sum)
effective_retention_raster.set(
global_col, global_row, working_retention_eff)
else:
LOGGER.error('outflow_weight_sum %s', outflow_weight_sum)
raise Exception("got to a cell that has no outflow!")
# search upslope to see if we need to push a cell on the stack
for i in range(8):
neighbor_col = col_offsets[i] + global_col
if neighbor_col < 0 or neighbor_col >= n_cols:
continue
neighbor_row = row_offsets[i] + global_row
if neighbor_row < 0 or neighbor_row >= n_rows:
continue
neighbor_outflow_dir = inflow_offsets[i]
neighbor_outflow_dir_mask = 1 << neighbor_outflow_dir
neighbor_process_flow_dir = <int>(
to_process_flow_directions_raster.get(
neighbor_col, neighbor_row))
if neighbor_process_flow_dir == 0:
# skip, due to loop invariant this must be a nodata pixel
continue
if neighbor_process_flow_dir & neighbor_outflow_dir_mask == 0:
# no outflow
continue
# mask out the outflow dir that this iteration processed
neighbor_process_flow_dir &= ~neighbor_outflow_dir_mask
to_process_flow_directions_raster.set(
neighbor_col, neighbor_row, neighbor_process_flow_dir)
if neighbor_process_flow_dir == 0:
# if 0 then all downslope have been processed,
# push on stack, otherwise another downslope pixel will
# pick it up
processing_stack.push(neighbor_row*n_cols + neighbor_col)
to_process_flow_directions_raster.close()
os.remove(to_process_flow_directions_path)
if algorithm == 'MFD':
run_effective_retention[MFD](
flow_direction_path.encode('utf-8'),
stream_path.encode('utf-8'),
retention_eff_lulc_path.encode('utf-8'),
crit_len_path.encode('utf-8'),
to_process_flow_directions_path.encode('utf-8'),
effective_retention_path.encode('utf-8'))
else: # D8
run_effective_retention[D8](
flow_direction_path.encode('utf-8'),
stream_path.encode('utf-8'),
retention_eff_lulc_path.encode('utf-8'),
crit_len_path.encode('utf-8'),
to_process_flow_directions_path.encode('utf-8'),
effective_retention_path.encode('utf-8'))

1
src/natcap/invest/pollination.py
View File

@ -24,6 +24,7 @@ from .unit_registry import u
LOGGER = logging.getLogger(__name__)
MODEL_SPEC = {
"model_id": "pollination",
"model_name": MODEL_METADATA["pollination"].model_title,
"pyname": MODEL_METADATA["pollination"].pyname,
"userguide": MODEL_METADATA["pollination"].userguide,

30
src/natcap/invest/recreation/_utils.py Normal file
View File

@ -0,0 +1,30 @@
from io import BytesIO
import numpy
def _numpy_dumps(numpy_array):
"""Safely pickle numpy array to string.
Args:
numpy_array (numpy.ndarray): arbitrary numpy array.
Returns:
A string representation of the array that can be loaded using
`numpy_loads`.
"""
with BytesIO() as file_stream:
numpy.save(file_stream, numpy_array, allow_pickle=False)
return file_stream.getvalue()
def _numpy_loads(queue_string):
"""Safely unpickle string to numpy array.
Args:
queue_string (str): binary string representing a pickled
numpy array.
Returns:
A numpy representation of ``binary_numpy_string``.
"""
with BytesIO(queue_string) as file_stream:
return numpy.load(file_stream)

138
src/natcap/invest/recreation/buffered_numpy_disk_map.py
View File

@ -5,10 +5,12 @@ import time
import collections
import os
import logging
import multiprocessing
import sqlite3
import numpy
from ._utils import _numpy_dumps, _numpy_loads
from .. import utils
@ -16,6 +18,25 @@ LOGGER = logging.getLogger(
'natcap.invest.recmodel_server.buffered_numpy_disk_map')
def _npy_append(filepath, array):
"""Append to a numpy array on disk without reading the entire array."""
with open(filepath, 'rb+') as file:
version = numpy.lib.format.read_magic(file)
header_tuple = numpy.lib.format._read_array_header(file, version)
header_dict = {
'shape': header_tuple[0],
'fortran_order': header_tuple[1],
'descr': numpy.lib.format.dtype_to_descr(header_tuple[2])
}
# update the shape because we intend to append array
n = header_dict['shape'][0] + array.size
header_dict['shape'] = (n, )
file.seek(0, 2) # go to end to append data
file.write(array)
file.seek(0, 0) # go to start to re-write header
numpy.lib.format._write_array_header(file, header_dict, version)
class BufferedNumpyDiskMap(object):
"""Persistent object to append and read numpy arrays to unique keys.
@ -25,9 +46,9 @@ class BufferedNumpyDiskMap(object):
files on disk to manage memory and persist between instantiations.
"""
_ARRAY_TUPLE_TYPE = numpy.dtype('datetime64[D],a4,f4,f4')
_ARRAY_TUPLE_TYPE = numpy.dtype('datetime64[D],S4,f4,f4')
def __init__(self, manager_filename, max_bytes_to_buffer):
def __init__(self, manager_filename, max_bytes_to_buffer, n_workers=1):
"""Create file manager object.
Args:
@ -36,10 +57,13 @@ class BufferedNumpyDiskMap(object):
binary data as needed.
max_bytes_to_buffer (int): number of bytes to hold in memory at
one time.
n_workers (int): if greater than 1, number of child processes to
use during flushes to disk.
Returns:
None
"""
self.n_workers = n_workers
self.manager_filename = manager_filename
self.manager_directory = os.path.dirname(manager_filename)
utils.make_directories([self.manager_directory])
@ -66,12 +90,56 @@ class BufferedNumpyDiskMap(object):
Returns:
None
"""
self.array_cache[array_id].append(array_data.copy())
self.array_cache[array_id].append(_numpy_dumps(array_data))
self.current_bytes_in_system += (
array_data.size * BufferedNumpyDiskMap._ARRAY_TUPLE_TYPE.itemsize)
if self.current_bytes_in_system > self.max_bytes_to_buffer:
self.flush()
def _write(self, array_id_list):
db_connection = sqlite3.connect(
self.manager_filename, detect_types=sqlite3.PARSE_DECLTYPES)
db_cursor = db_connection.cursor()
insert_list = []
if not isinstance(array_id_list, list):
array_id_list = [array_id_list]
for array_id in array_id_list:
array_deque = collections.deque(
_numpy_loads(x) for x in self.array_cache[array_id])
# try to get data if it's there
db_cursor.execute(
"""SELECT (array_path) FROM array_table
where array_id=? LIMIT 1""", [array_id])
array_path = db_cursor.fetchone()
if array_path is not None:
_npy_append(os.path.join(self.manager_directory, array_path[0]),
numpy.concatenate(array_deque))
array_deque = None
else:
# make a random filename and put it one directory deep named
# off the last two characters in the filename
array_filename = uuid.uuid4().hex + '.npy'
# -6:-4 skips the extension and gets the last 2 characters
array_subdirectory = array_filename[-6:-4]
array_directory = os.path.join(
self.manager_directory, array_subdirectory)
try:
# multiple processes could be trying to make this dir
# so use a try/except instead of checking for existence
os.mkdir(array_directory)
except FileExistsError:
pass
array_path = os.path.join(array_directory, array_filename)
# save the file
array_data = numpy.concatenate(array_deque)
array_deque = None
numpy.save(array_path, array_data)
insert_list.append(
(array_id,
os.path.join(array_subdirectory, array_filename)))
db_connection.close()
return insert_list
def flush(self):
"""Method to flush data in memory to disk."""
start_time = time.time()
@ -79,43 +147,25 @@ class BufferedNumpyDiskMap(object):
'Flushing %d bytes in %d arrays', self.current_bytes_in_system,
len(self.array_cache))
array_id_list = list(self.array_cache)
n_workers = self.n_workers \
if self.n_workers <= len(array_id_list) else len(array_id_list)
LOGGER.debug(f'N_WORKERS for flush: {n_workers}')
if n_workers > 1:
with multiprocessing.Pool(processes=self.n_workers) as pool:
insert_list_of_lists = pool.map(self._write, array_id_list)
pool.close()
pool.join()
insert_list = [x for xs in insert_list_of_lists for x in xs]
else:
insert_list = self._write(array_id_list)
for array_id in array_id_list:
del self.array_cache[array_id]
db_connection = sqlite3.connect(
self.manager_filename, detect_types=sqlite3.PARSE_DECLTYPES)
db_cursor = db_connection.cursor()
# get all the array data to append at once
insert_list = []
while len(self.array_cache) > 0:
array_id = next(iter(self.array_cache.keys()))
array_deque = self.array_cache.pop(array_id)
# try to get data if it's there
db_cursor.execute(
"""SELECT (array_path) FROM array_table
where array_id=? LIMIT 1""", [array_id])
array_path = db_cursor.fetchone()
if array_path is not None:
# cache gets wiped at end so okay to use same deque
array_deque.append(numpy.load(array_path[0]))
array_data = numpy.concatenate(array_deque)
array_deque = None
numpy.save(array_path[0], array_data)
else:
# otherwise directly write
# make a random filename and put it one directory deep named
# off the last two characters in the filename
array_filename = uuid.uuid4().hex + '.npy'
# -6:-4 skips the extension and gets the last 2 characters
array_directory = os.path.join(
self.manager_directory, array_filename[-6:-4])
if not os.path.isdir(array_directory):
os.mkdir(array_directory)
array_path = os.path.join(array_directory, array_filename)
# save the file
array_data = numpy.concatenate(array_deque)
array_deque = None
numpy.save(array_path, array_data)
insert_list.append((array_id, array_path))
db_cursor.executemany(
"""INSERT INTO array_table
(array_id, array_path)
@ -149,13 +199,15 @@ class BufferedNumpyDiskMap(object):
db_connection.close()
if array_path is not None:
array_data = numpy.load(array_path[0])
array_data = numpy.load(
os.path.join(self.manager_directory, array_path[0]))
else:
array_data = numpy.empty(
0, dtype=BufferedNumpyDiskMap._ARRAY_TUPLE_TYPE)
if len(self.array_cache[array_id]) > 0:
local_deque = collections.deque(self.array_cache[array_id])
local_deque = collections.deque(
_numpy_loads(x) for x in self.array_cache[array_id])
local_deque.append(array_data)
array_data = numpy.concatenate(local_deque)
@ -171,10 +223,11 @@ class BufferedNumpyDiskMap(object):
[array_id])
array_path = db_cursor.fetchone()
if array_path is not None:
os.remove(array_path[0])
array_abs_path = os.path.join(self.manager_directory, array_path[0])
os.remove(array_abs_path)
try:
# attempt to remove the directory if it's empty
os.rmdir(os.path.dirname(array_path[0]))
os.rmdir(os.path.dirname(array_abs_path))
except OSError:
# it's not empty, not a big deal
pass
@ -185,8 +238,7 @@ class BufferedNumpyDiskMap(object):
db_connection.close()
# delete the cache and update cache size
# The * 12 comes from the fact that the array is an 'a4 f4 f4'
self.current_bytes_in_system -= (
sum([x.size for x in self.array_cache[array_id]]) *
sum([_numpy_loads(x).size for x in self.array_cache[array_id]]) *
BufferedNumpyDiskMap._ARRAY_TUPLE_TYPE.itemsize)
del self.array_cache[array_id]

38
src/natcap/invest/recreation/out_of_core_quadtree.pyx
View File

@ -36,7 +36,7 @@ class OutOfCoreQuadTree(object):
def __init__(
self, bounding_box, max_points_per_node, max_node_depth,
quad_tree_storage_dir, node_depth=0, node_data_manager=None,
pickle_filename=None):
pickle_filename=None, n_workers=1):
"""Make a new quadtree node with a given initial_bounding_box range.
Args:
@ -51,6 +51,8 @@ class OutOfCoreQuadTree(object):
to store the node data across the entire quadtree
pickle_filename (string): name of file on disk which to pickle the
tree to during a flush
n_workers (int): if greater than 1, number of child processes to
use during flushes to disk.
Returns:
None
@ -66,7 +68,8 @@ class OutOfCoreQuadTree(object):
if node_data_manager is None:
self.node_data_manager = (
buffered_numpy_disk_map.BufferedNumpyDiskMap(
pickle_filename+'.db', MAX_BYTES_TO_BUFFER))
pickle_filename+'.db', MAX_BYTES_TO_BUFFER,
n_workers=n_workers))
else:
self.node_data_manager = node_data_manager
@ -108,7 +111,7 @@ class OutOfCoreQuadTree(object):
feature.SetField('bb_box', str(self.bounding_box))
ogr_polygon_layer.CreateFeature(feature)
else:
for node_index in xrange(4):
for node_index in range(4):
self.nodes[node_index].build_node_shapes(ogr_polygon_layer)
def _get_points_from_node(self):
@ -260,13 +263,13 @@ class OutOfCoreQuadTree(object):
"""Return the number of nodes in the quadtree"""
if self.is_leaf:
return 1
return sum([self.nodes[index].n_nodes() for index in xrange(4)]) + 1
return sum([self.nodes[index].n_nodes() for index in range(4)]) + 1
def n_points(self):
"""Return the number of nodes in the quadtree"""
"""Return the number of points in the quadtree"""
if self.is_leaf:
return self.n_points_in_node
return sum([self.nodes[index].n_points() for index in xrange(4)])
return sum([self.nodes[index].n_points() for index in range(4)])
def get_intersecting_points_in_polygon(self, shapely_polygon):
"""Return the points contained in `shapely_prepared_polygon`.
@ -305,7 +308,7 @@ class OutOfCoreQuadTree(object):
elif shapely_polygon.intersects(bounding_polygon):
# combine results of children
result_deque = collections.deque()
for node_index in xrange(4):
for node_index in range(4):
result_deque.extend(
self.nodes[node_index].get_intersecting_points_in_polygon(
shapely_polygon))
@ -339,12 +342,31 @@ class OutOfCoreQuadTree(object):
return point_list
else:
point_list = numpy.empty(0, dtype=_ARRAY_TUPLE_TYPE)
for node_index in xrange(4):
for node_index in range(4):
point_list = numpy.concatenate((
self.nodes[node_index].get_intersecting_points_in_bounding_box(
bounding_box), point_list))
return point_list
def estimate_points_in_bounding_box(self, bounding_box):
"""Count points in nodes intersecting a bounding_box.
Args:
bounding_box (list): of the form [xmin, ymin, xmax, ymax]
Returns:
int
"""
if not self._bounding_box_intersect(bounding_box):
return 0
if self.is_leaf:
return self.n_points_in_node
else:
return sum([
self.nodes[index].estimate_points_in_bounding_box(bounding_box)
for index in range(4)])
cdef _sort_list_to_quads(
numpy.ndarray point_list, int left_bound, int right_bound,

734
src/natcap/invest/recreation/recmodel_client.py
View File

File diff suppressed because it is too large Load Diff

1105
src/natcap/invest/recreation/recmodel_server.py
View File

File diff suppressed because it is too large Load Diff

27
src/natcap/invest/recreation/recmodel_workspace_fetcher.py
View File

@ -4,16 +4,16 @@ import os
import logging
import urllib
import Pyro4
import Pyro5
import Pyro5.api
from natcap.invest.recreation import recmodel_client
from .. import utils
LOGGER = logging.getLogger('natcap.invest.recmodel_client')
# This URL is a NatCap global constant
RECREATION_SERVER_URL = 'http://data.naturalcapitalproject.org/server_registry/invest_recreation_model/'
# this serializer lets us pass null bytes in strings unlike the default
Pyro4.config.SERIALIZER = 'marshal'
Pyro5.config.SERIALIZER = 'marshal'
def execute(args):
@ -28,6 +28,7 @@ def execute(args):
args['hostname'] (string): FQDN to recreation server
args['port'] (string or int): port on hostname for recreation server
args['workspace_id'] (string): workspace identifier
args['server_id'] (string): one of ('flickr', 'twitter')
Returns:
None
@ -37,20 +38,20 @@ def execute(args):
# in case the user defines a hostname
if 'hostname' in args:
path = "PYRO:natcap.invest.recreation@%s:%s" % (
server_url = "PYRO:natcap.invest.recreation@%s:%s" % (
args['hostname'], args['port'])
else:
# else use a well known path to get active server
path = urllib.urlopen(RECREATION_SERVER_URL).read().rstrip()
server_url = urllib.urlopen(recmodel_client.SERVER_URL).read().rstrip()
LOGGER.info("contacting server")
recmodel_server = Pyro4.Proxy(path)
recmodel_manager = Pyro5.api.Proxy(server_url)
LOGGER.info("sending id request %s", args['workspace_id'])
workspace_aoi_binary = recmodel_server.fetch_workspace_aoi(
args['workspace_id'])
zip_binary = recmodel_manager.fetch_aoi_workspaces(
args['workspace_id'], args['server_id'])
# unpack result
with open(os.path.join(
output_dir, '%s.zip' % args['workspace_id']), 'wb') as file:
file.write(workspace_aoi_binary)
LOGGER.info("fetched aoi")
output_dir,
f"{args['server_id']}_{args['workspace_id']}.zip"), 'wb') as file:
file.write(zip_binary)
LOGGER.info(f"fetched {args['server_id']}_{args['workspace_id']}.zip")

1
src/natcap/invest/routedem.py
View File

@ -18,6 +18,7 @@ LOGGER = logging.getLogger(__name__)
INVALID_BAND_INDEX_MSG = gettext('Must be between 1 and {maximum}')
MODEL_SPEC = {
"model_id": "routedem",
"model_name": MODEL_METADATA["routedem"].model_title,
"pyname": MODEL_METADATA["routedem"].pyname,
"userguide": MODEL_METADATA["routedem"].userguide,

39
src/natcap/invest/scenario_gen_proximity.py
View File

@ -30,6 +30,7 @@ MISSING_CONVERT_OPTION_MSG = gettext(
'must be selected')
MODEL_SPEC = {
"model_id": "scenario_generator_proximity",
"model_name": MODEL_METADATA["scenario_generator_proximity"].model_title,
"pyname": MODEL_METADATA["scenario_generator_proximity"].pyname,
"userguide": MODEL_METADATA["scenario_generator_proximity"].userguide,
@ -69,7 +70,7 @@ MODEL_SPEC = {
"regexp": "[0-9 ]+",
"about": gettext(
"A space-separated list of LULC codes that can be "
"converted to be converted to agriculture."),
"converted to agriculture."),
"name": gettext("convertible landcover codes")
},
"n_fragmentation_steps": {
@ -122,11 +123,15 @@ MODEL_SPEC = {
"nearest_to_edge.csv": {
"about": gettext(
"Table of land cover classes and the amount of each that was converted for the nearest-to-edge conversion scenario."),
"index_col": "lucode",
"index_col": "original lucode",
"columns": {
"lucode": {
"original lucode": {
"type": "integer",
"about": "LULC code of the land cover class"
"about": "Original LULC code of the land cover class"
},
"replacement lucode": {
"type": "integer",
"about": "LULC code to which habitat was converted"
},
"area converted (Ha)": {
"type": "number",
@ -141,12 +146,16 @@ MODEL_SPEC = {
},
"farthest_from_edge.csv": {
"about": gettext(
"Table of land cover classes and the amount of each that was converted for the nearest-to-edge conversion scenario."),
"index_col": "lucode",
"Table of land cover classes and the amount of each that was converted for the farthest-from-edge conversion scenario."),
"index_col": "original lucode",
"columns": {
"lucode": {
"original lucode": {
"type": "integer",
"about": "LULC code of the land cover class"
"about": "Original LULC code of the land cover class"
},
"replacement lucode": {
"type": "integer",
"about": "LULC code to which habitat was converted"
},
"area converted (Ha)": {
"type": "number",
@ -555,7 +564,7 @@ def _convert_landscape(
output_landscape_raster_path, replacement_lucode, stats_cache,
score_weight)
_log_stats(stats_cache, pixel_area_ha, stats_path)
_log_stats(stats_cache, replacement_lucode, pixel_area_ha, stats_path)
try:
shutil.rmtree(temp_dir)
except OSError:
@ -563,12 +572,13 @@ def _convert_landscape(
"Could not delete temporary working directory '%s'", temp_dir)
def _log_stats(stats_cache, pixel_area, stats_path):
def _log_stats(stats_cache, replacement_lucode, pixel_area, stats_path):
"""Write pixel change statistics to a file in tabular format.
Args:
stats_cache (dict): a dictionary mapping pixel lucodes to number of
pixels changed
replacement_lucode (int): lucode to which habitat was converted
pixel_area (float): size of pixels in hectares so an area column can
be generated
stats_path (string): path to a csv file that the table should be
@ -579,11 +589,14 @@ def _log_stats(stats_cache, pixel_area, stats_path):
"""
with open(stats_path, 'w') as csv_output_file:
csv_output_file.write('lucode,area converted (Ha),pixels converted\n')
csv_output_file.write(
'original lucode,replacement lucode,'
'area converted (Ha),pixels converted\n')
for lucode in sorted(stats_cache):
csv_output_file.write(
'%s,%s,%s\n' % (
lucode, stats_cache[lucode] * pixel_area,
'%s,%s,%s,%s\n' % (
lucode, replacement_lucode,
stats_cache[lucode] * pixel_area,
stats_cache[lucode]))

73
src/natcap/invest/scenic_quality/LRUCache.h
View File

@ -1,73 +0,0 @@
#ifndef __LRUCACHE_H_INCLUDED__
#define __LRUCACHE_H_INCLUDED__
#include <list>
#include <map>
#include <assert.h>
using namespace std;
template <class KEY_T, class VAL_T,
typename ListIter = typename list< pair<KEY_T,VAL_T> >::iterator,
typename MapIter = typename map<KEY_T, ListIter>::iterator > class LRUCache{
private:
// item_list keeps track of the order of which elements have been accessed
// element at begin is most recent, element at end is least recent.
// first element in the pair is its key while the second is the element
list< pair<KEY_T,VAL_T> > item_list;
// item_map maps an element's key to its location in the `item_list`
// used to make lookups O(log n) time
map<KEY_T, ListIter> item_map;
size_t cache_size;
private:
void clean(list< pair<KEY_T, VAL_T> > &removed_value_list){
while(item_map.size()>cache_size){
ListIter last_it = item_list.end(); last_it --;
removed_value_list.push_back(
make_pair(last_it->first, last_it->second));
item_map.erase(last_it->first);
item_list.pop_back();
}
};
public:
LRUCache(int cache_size_):cache_size(cache_size_){
;
};
ListIter begin() {
return item_list.begin();
}
ListIter end() {
return item_list.end();
}
void put(
const KEY_T &key, const VAL_T &val,
list< pair<KEY_T, VAL_T> > &removed_value_list) {
MapIter it = item_map.find(key);
if(it != item_map.end()){
// it's already in the cache, delete the location in the item
// list and in the lookup map
item_list.erase(it->second);
item_map.erase(it);
}
// insert a new item in the front since it's most recently used
item_list.push_front(make_pair(key,val));
// record its iterator in the map
item_map.insert(make_pair(key, item_list.begin()));
// possibly remove any elements that have exceeded the cache size
return clean(removed_value_list);
};
bool exist(const KEY_T &key){
return (item_map.count(key)>0);
};
VAL_T& get(const KEY_T &key){
MapIter it = item_map.find(key);
assert(it!=item_map.end());
// move the element to the front of the list
item_list.splice(item_list.begin(), item_list, it->second);
return it->second->second;
};
};
#endif

1
src/natcap/invest/scenic_quality/scenic_quality.py
View File

@ -48,6 +48,7 @@ _INTERMEDIATE_BASE_FILES = {
}
MODEL_SPEC = {
"model_id": "scenic_quality",
"model_name": MODEL_METADATA["scenic_quality"].model_title,
"pyname": MODEL_METADATA["scenic_quality"].pyname,
"userguide": MODEL_METADATA["scenic_quality"].userguide,

371
src/natcap/invest/scenic_quality/viewshed.pyx
View File

@ -30,12 +30,8 @@ from osgeo import gdal
from osgeo import osr
import shapely.geometry
from .. import utils
from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cython.operator cimport dereference as deref
from cython.operator cimport preincrement as inc
from libc.time cimport time_t
from libc.time cimport time as ctime
from libcpp.list cimport list as clist
from libcpp.set cimport set as cset
from libcpp.deque cimport deque
from libcpp.pair cimport pair
@ -43,7 +39,8 @@ from libcpp.queue cimport queue
from libc cimport math
cimport numpy
cimport cython
from ..managed_raster.managed_raster cimport ManagedRaster
from ..managed_raster.managed_raster cimport is_close
LOGGER = logging.getLogger(__name__)
LOGGER.addHandler(logging.NullHandler())
@ -159,20 +156,6 @@ cdef inline long lmin(long a, long b):
return b
# this is a least recently used cache written in C++ in an external file,
# exposing here so _ManagedRaster can use it.
# Copied from src/pygeoprocessing/routing/routing.pyx,
# revision 891288683889237cfd3a3d0a1f09483c23489fca.
cdef extern from "LRUCache.h":
cdef cppclass LRUCache[KEY_T, VAL_T]:
LRUCache(int)
void put(KEY_T&, VAL_T&, clist[pair[KEY_T,VAL_T]]&)
clist[pair[KEY_T,VAL_T]].iterator begin()
clist[pair[KEY_T,VAL_T]].iterator end()
bint exist(KEY_T &)
VAL_T get(KEY_T &)
# exposing stl::priority_queue so we can have all 3 template arguments so
# we can pass a different Compare functor
cdef extern from "<queue>" namespace "std":
@ -224,310 +207,9 @@ cdef inline double pixel_dist(long ix_source, long ix_target, long iy_source, lo
lmax(ix_source, ix_target)-lmin(ix_source, ix_target),
lmax(iy_source, iy_target)-lmin(iy_source, iy_target))
# Number of raster blocks to hold in memory at once per Managed Raster
cdef int MANAGED_RASTER_N_BLOCKS = 2**6
# The nodata value for visibility rasters
cdef int VISIBILITY_NODATA = 255
# this ctype is used to store the block ID and the block buffer as one object
# inside Managed Raster
ctypedef pair[int, double*] BlockBufferPair
# a class to allow fast random per-pixel access to a raster for both setting
# and reading pixels. Copied from src/pygeoprocessing/routing/routing.pyx,
# revision 891288683889237cfd3a3d0a1f09483c23489fca.
cdef class _ManagedRaster:
cdef LRUCache[int, double*]* lru_cache
cdef cset[int] dirty_blocks
cdef int block_xsize
cdef int block_ysize
cdef int block_xmod
cdef int block_ymod
cdef int block_xbits
cdef int block_ybits
cdef long raster_x_size
cdef long raster_y_size
cdef int block_nx
cdef int block_ny
cdef int write_mode
cdef bytes raster_path
cdef int band_id
cdef int closed
def __cinit__(self, raster_path, band_id, write_mode):
"""Create new instance of Managed Raster.
Args:
raster_path (char*): path to raster that has block sizes that are
powers of 2. If not, an exception is raised.
band_id (int): which band in `raster_path` to index. Uses GDAL
notation that starts at 1.
write_mode (boolean): if true, this raster is writable and dirty
memory blocks will be written back to the raster as blocks
are swapped out of the cache or when the object deconstructs.
Returns:
None.
"""
raster_info = pygeoprocessing.get_raster_info(raster_path)
self.raster_x_size, self.raster_y_size = raster_info['raster_size']
self.block_xsize, self.block_ysize = raster_info['block_size']
self.block_xmod = self.block_xsize-1
self.block_ymod = self.block_ysize-1
if not (1 <= band_id <= raster_info['n_bands']):
err_msg = (
"Error: band ID (%s) is not a valid band number. "
"This exception is happening in Cython, so it will cause a "
"hard seg-fault, but it's otherwise meant to be a "
"ValueError." % (band_id))
print(err_msg)
raise ValueError(err_msg)
self.band_id = band_id
if (self.block_xsize & (self.block_xsize - 1) != 0) or (
self.block_ysize & (self.block_ysize - 1) != 0):
# If inputs are not a power of two, this will at least print
# an error message. Unfortunately with Cython, the exception will
# present itself as a hard seg-fault, but I'm leaving the
# ValueError in here at least for readability.
err_msg = (
"Error: Block size is not a power of two: "
"block_xsize: %d, %d, %s. This exception is happening"
"in Cython, so it will cause a hard seg-fault, but it's"
"otherwise meant to be a ValueError." % (
self.block_xsize, self.block_ysize, raster_path))
print(err_msg)
raise ValueError(err_msg)
self.block_xbits = numpy.log2(self.block_xsize)
self.block_ybits = numpy.log2(self.block_ysize)
self.block_nx = (
self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize
self.block_ny = (
self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize
self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS)
self.raster_path = <bytes> raster_path
self.write_mode = write_mode
self.closed = 0
def __dealloc__(self):
"""Deallocate _ManagedRaster.
This operation manually frees memory from the LRUCache and writes any
dirty memory blocks back to the raster if `self.write_mode` is True.
"""
self.close()
def close(self):
"""Close the _ManagedRaster and free up resources.
This call writes any dirty blocks to disk, frees up the memory
allocated as part of the cache, and frees all GDAL references.
Any subsequent calls to any other functions in _ManagedRaster will
have undefined behavior.
"""
if self.closed:
return
self.closed = 1
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array = numpy.empty(
(self.block_ysize, self.block_xsize))
cdef double *double_buffer
cdef int block_xi
cdef int block_yi
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize
cdef int win_ysize
# we need the offsets to subtract from global indexes for cached array
cdef int xoff
cdef int yoff
cdef clist[BlockBufferPair].iterator it = self.lru_cache.begin()
cdef clist[BlockBufferPair].iterator end = self.lru_cache.end()
if not self.write_mode:
while it != end:
# write the changed value back if desired
PyMem_Free(deref(it).second)
inc(it)
return
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
# if we get here, we're in write_mode
cdef cset[int].iterator dirty_itr
while it != end:
double_buffer = deref(it).second
block_index = deref(it).first
# write to disk if block is dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for
# cached array
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
# clip window sizes if necessary
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = (
double_buffer[
(yi_copy << self.block_xbits) + xi_copy])
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
inc(it)
raster_band.FlushCache()
raster_band = None
raster = None
cdef inline void set(self, long xi, long yi, double value):
"""Set the pixel at `xi,yi` to `value`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))] = value
if self.write_mode:
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr == self.dirty_blocks.end():
self.dirty_blocks.insert(block_index)
cdef inline double get(self, long xi, long yi):
"""Return the value of the pixel at `xi,yi`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
return self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))]
cdef void _load_block(self, int block_index) except *:
cdef int block_xi = block_index % self.block_nx
cdef int block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for cached array
cdef int xoff = block_xi << self.block_xbits
cdef int yoff = block_yi << self.block_ybits
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array
cdef double *double_buffer
cdef clist[BlockBufferPair] removed_value_list
# determine the block aligned xoffset for read as array
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize = self.block_xsize
cdef int win_ysize = self.block_ysize
# load a new block
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (yoff+win_ysize - self.raster_y_size)
raster = gdal.OpenEx(self.raster_path, gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = raster_band.ReadAsArray(
xoff=xoff, yoff=yoff, win_xsize=win_xsize,
win_ysize=win_ysize).astype(
numpy.float64)
raster_band = None
raster = None
double_buffer = <double*>PyMem_Malloc(
(sizeof(double) << self.block_xbits) * win_ysize)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
double_buffer[(yi_copy<<self.block_xbits)+xi_copy] = (
block_array[yi_copy, xi_copy])
self.lru_cache.put(
<int>block_index, <double*>double_buffer, removed_value_list)
if self.write_mode:
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = numpy.empty(
(self.block_ysize, self.block_xsize), dtype=numpy.double)
while not removed_value_list.empty():
# write the changed value back if desired
double_buffer = removed_value_list.front().second
if self.write_mode:
block_index = removed_value_list.front().first
# write back the block if it's dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = double_buffer[
(yi_copy << self.block_xbits) + xi_copy]
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
removed_value_list.pop_front()
if self.write_mode:
raster_band = None
raster = None
@cython.binding(True)
@cython.boundscheck(False)
@cython.cdivision(True)
@ -616,12 +298,12 @@ def viewshed(dem_raster_path_band,
band_to_array_index = dem_raster_path_band[1] - 1
nodata_value = dem_raster_info['nodata'][band_to_array_index]
if viewpoint_elevation == nodata_value:
raise LookupError('Viewpoint is over nodata')
# Need to handle the case where the nodata value is not defined.
if nodata_value is None:
nodata_value = IMPROBABLE_NODATA
elif is_close(viewpoint_elevation, nodata_value):
raise LookupError('Viewpoint is over nodata')
cdef double nodata = nodata_value
# Verify that pixels are very close to square. The Wang et al algorithm
@ -634,8 +316,8 @@ def viewshed(dem_raster_path_band,
(pixel_xsize, pixel_ysize))
# Verify that the block sizes are powers of 2 and are square.
# This is needed for the _ManagedRaster classes. If this is not asserted
# here, the _ManagedRaster classes will crash with a segfault.
# This is needed for the ManagedRaster classes. If this is not asserted
# here, the ManagedRaster classes will crash with a segfault.
block_xsize, block_ysize = dem_raster_info['block_size']
if (block_xsize & (block_xsize - 1) != 0 or (
block_ysize & (block_ysize - 1) != 0)) or (
@ -669,12 +351,12 @@ def viewshed(dem_raster_path_band,
raster_driver_creation_tuple=BYTE_GTIFF_CREATION_OPTIONS)
# LRU-cached rasters for easier access to individual pixels.
cdef _ManagedRaster dem_managed_raster = (
_ManagedRaster(dem_raster_path_band[0], dem_raster_path_band[1], 0))
cdef _ManagedRaster aux_managed_raster = (
_ManagedRaster(aux_filepath, 1, 1))
cdef _ManagedRaster visibility_managed_raster = (
_ManagedRaster(visibility_filepath, 1, 1))
cdef ManagedRaster dem_managed_raster = (
ManagedRaster(dem_raster_path_band[0].encode('utf-8'), dem_raster_path_band[1], 0))
cdef ManagedRaster aux_managed_raster = (
ManagedRaster(aux_filepath.encode('utf-8'), 1, 1))
cdef ManagedRaster visibility_managed_raster = (
ManagedRaster(visibility_filepath.encode('utf-8'), 1, 1))
# get the pixel size in terms of meters.
dem_srs = osr.SpatialReference()
@ -830,7 +512,7 @@ def viewshed(dem_raster_path_band,
adjusted_dem_height = target_dem_height - adjustment
if (adjusted_dem_height >= z and
target_distance < max_visible_radius and
target_dem_height != nodata):
not is_close(target_dem_height, nodata)):
visibility_managed_raster.set(ix_target, iy_target, 1)
aux_managed_raster.set(ix_target, iy_target, adjusted_dem_height)
else:
@ -957,22 +639,21 @@ def viewshed(dem_raster_path_band,
# than or equal to the minimum-visible height AND is closer than the
# maximum visible radius.
target_dem_height = dem_managed_raster.get(n, m)
adjusted_dem_height = dem_managed_raster.get(n, m) - adjustment
if (adjusted_dem_height >= z and
target_pixel.distance_to_viewpoint < max_visible_radius and
target_dem_height != nodata):
visibility_managed_raster.set(n, m, 1)
adjusted_dem_height = target_dem_height - adjustment
# If it's close enough to nodata to be interpreted as nodata,
# consider it to be nodata. Nodata implies that visibility is
# undefined ... which it is, since there's no defined DEM value for
# this pixel.
if is_close(target_dem_height, nodata):
visibility_managed_raster.set(n, m, VISIBILITY_NODATA)
aux_managed_raster.set(n, m, z)
elif (adjusted_dem_height >= z and
target_pixel.distance_to_viewpoint < max_visible_radius):
visibility_managed_raster.set(n, m, 1) # the pixel is visible
aux_managed_raster.set(n, m, adjusted_dem_height)
else:
# If it's close enough to nodata to be interpreted as nodata,
# consider it to be nodata. Nodata implies that visibility is
# undefined ... which it is, since there's no defined DEM value for
# this pixel.
if math.fabs(target_dem_height - nodata) <= 1.0e-7:
visibility_managed_raster.set(n, m, VISIBILITY_NODATA)
else:
# If we're not over nodata, then the pixel isn't visible.
visibility_managed_raster.set(n, m, 0)
visibility_managed_raster.set(n, m, 0) # the pixel isn't visible
aux_managed_raster.set(n, m, z)
pixels_touched += 1

73
src/natcap/invest/sdr/LRUCache.h
View File

@ -1,73 +0,0 @@
#ifndef __LRUCACHE_H_INCLUDED__
#define __LRUCACHE_H_INCLUDED__
#include <list>
#include <map>
#include <assert.h>
using namespace std;
template <class KEY_T, class VAL_T,
typename ListIter = typename list< pair<KEY_T,VAL_T> >::iterator,
typename MapIter = typename map<KEY_T, ListIter>::iterator > class LRUCache{
private:
// item_list keeps track of the order of which elements have been accessed
// element at begin is most recent, element at end is least recent.
// first element in the pair is its key while the second is the element
list< pair<KEY_T,VAL_T> > item_list;
// item_map maps an element's key to its location in the `item_list`
// used to make lookups O(log n) time
map<KEY_T, ListIter> item_map;
size_t cache_size;
private:
void clean(list< pair<KEY_T, VAL_T> > &removed_value_list){
while(item_map.size()>cache_size){
ListIter last_it = item_list.end(); last_it --;
removed_value_list.push_back(
make_pair(last_it->first, last_it->second));
item_map.erase(last_it->first);
item_list.pop_back();
}
};
public:
LRUCache(int cache_size_):cache_size(cache_size_){
;
};
ListIter begin() {
return item_list.begin();
}
ListIter end() {
return item_list.end();
}
void put(
const KEY_T &key, const VAL_T &val,
list< pair<KEY_T, VAL_T> > &removed_value_list) {
MapIter it = item_map.find(key);
if(it != item_map.end()){
// it's already in the cache, delete the location in the item
// list and in the lookup map
item_list.erase(it->second);
item_map.erase(it);
}
// insert a new item in the front since it's most recently used
item_list.push_front(make_pair(key,val));
// record its iterator in the map
item_map.insert(make_pair(key, item_list.begin()));
// possibly remove any elements that have exceeded the cache size
return clean(removed_value_list);
};
bool exist(const KEY_T &key){
return (item_map.count(key)>0);
};
VAL_T& get(const KEY_T &key){
MapIter it = item_map.find(key);
assert(it!=item_map.end());
// move the element to the front of the list
item_list.splice(item_list.begin(), item_list, it->second);
return it->second->second;
};
};
#endif

218
src/natcap/invest/sdr/sdr.py
View File

@ -29,6 +29,7 @@ from . import sdr_core
LOGGER = logging.getLogger(__name__)
MODEL_SPEC = {
"model_id": "sdr",
"model_name": MODEL_METADATA["sdr"].model_title,
"pyname": MODEL_METADATA["sdr"].pyname,
"userguide": MODEL_METADATA["sdr"].userguide,
@ -140,27 +141,28 @@ MODEL_SPEC = {
"watershed. Pixels with 1 are drainages and are treated like "
"streams. Pixels with 0 are not drainages."),
"name": gettext("drainages")
}
},
**spec_utils.FLOW_DIR_ALGORITHM
},
"outputs": {
"avoided_erosion.tif": {
"about": "The contribution of vegetation to keeping soil from eroding from each pixel. (Eq. (82))",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"avoided_export.tif": {
"about": "The contribution of vegetation to keeping erosion from entering a stream. This combines local/on-pixel sediment retention with trapping of erosion from upslope of the pixel. (Eq. (83))",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"rkls.tif": {
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}},
"about": "Total potential soil loss per pixel in the original land cover from the RKLS equation. Equivalent to the soil loss for bare soil. (Eq. (68), without applying the C or P factors)."
},
@ -168,14 +170,14 @@ MODEL_SPEC = {
"about": "The total amount of sediment deposited on the pixel from upslope sources as a result of trapping. (Eq. (80))",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"sed_export.tif": {
"about": "The total amount of sediment exported from each pixel that reaches the stream. (Eq. (76))",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"stream.tif": spec_utils.STREAM,
@ -185,10 +187,10 @@ MODEL_SPEC = {
"bands": {1: {"type": "integer"}}
},
"usle.tif": {
"about": "Total potential soil loss per pixel in the original land cover calculated from the USLE equation. (Eq. (68))",
"about": "Total potential soil loss per hectare in the original land cover calculated from the USLE equation. (Eq. (68))",
"bands": {1: {
"type": "number",
"units": u.metric_ton/u.pixel
"units": u.metric_ton/u.hectare
}}
},
"watershed_results_sdr.shp": {
@ -681,23 +683,68 @@ def execute(args):
dependent_task_list=[slope_task],
task_name='threshold slope')
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(f_reg['pit_filled_dem_path'], 1),
f_reg['flow_direction_path']),
target_path_list=[f_reg['flow_direction_path']],
dependent_task_list=[pit_fill_task],
task_name='flow direction calculation')
if args['flow_dir_algorithm'] == 'MFD':
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(f_reg['pit_filled_dem_path'], 1),
f_reg['flow_direction_path']),
target_path_list=[f_reg['flow_direction_path']],
dependent_task_list=[pit_fill_task],
task_name='flow direction calculation')
flow_accumulation_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accumulation calculation')
flow_accumulation_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accumulation calculation')
stream_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(f_reg['flow_accumulation_path'], 1),
(f_reg['flow_direction_path'], 1),
float(args['threshold_flow_accumulation']),
f_reg['stream_path']),
kwargs={'trace_threshold_proportion': 0.7},
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accumulation_task],
task_name='extract streams')
d_dn_func = pygeoprocessing.routing.distance_to_channel_mfd
else:
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_d8,
args=(
(f_reg['pit_filled_dem_path'], 1),
f_reg['flow_direction_path']),
target_path_list=[f_reg['flow_direction_path']],
dependent_task_list=[pit_fill_task],
task_name='flow direction calculation')
flow_accumulation_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_d8,
args=(
(f_reg['flow_direction_path'], 1),
f_reg['flow_accumulation_path']),
target_path_list=[f_reg['flow_accumulation_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accumulation calculation')
stream_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_d8,
kwargs=dict(
flow_accum_raster_path_band=(f_reg['flow_accumulation_path'], 1),
flow_threshold=float(args['threshold_flow_accumulation']),
target_stream_raster_path=f_reg['stream_path']),
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accumulation_task],
task_name='extract streams')
d_dn_func = pygeoprocessing.routing.distance_to_channel_d8
ls_factor_task = task_graph.add_task(
func=_calculate_ls_factor,
@ -711,18 +758,6 @@ def execute(args):
flow_accumulation_task, slope_task],
task_name='ls factor calculation')
stream_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(f_reg['flow_accumulation_path'], 1),
(f_reg['flow_direction_path'], 1),
float(args['threshold_flow_accumulation']),
f_reg['stream_path']),
kwargs={'trace_threshold_proportion': 0.7},
target_path_list=[f_reg['stream_path']],
dependent_task_list=[flow_accumulation_task],
task_name='extract streams')
if drainage_present:
drainage_task = task_graph.add_task(
func=pygeoprocessing.raster_map,
@ -796,14 +831,17 @@ def execute(args):
's_bar')]:
bar_task = task_graph.add_task(
func=_calculate_bar_factor,
args=(
f_reg['flow_direction_path'], factor_path,
f_reg['flow_accumulation_path'],
accumulation_path, out_bar_path),
kwargs=dict(
flow_direction_path=f_reg['flow_direction_path'],
factor_path=factor_path,
flow_accumulation_path=f_reg['flow_accumulation_path'],
accumulation_path=accumulation_path,
out_bar_path=out_bar_path,
flow_dir_algorithm=args['flow_dir_algorithm']),
target_path_list=[accumulation_path, out_bar_path],
dependent_task_list=[
factor_task, flow_accumulation_task, flow_dir_task],
task_name='calculate %s' % bar_id)
task_name=f'calculate {bar_id}')
bar_task_map[bar_id] = bar_task
d_up_task = task_graph.add_task(
@ -829,7 +867,7 @@ def execute(args):
task_name='calculate inverse ws factor')
d_dn_task = task_graph.add_task(
func=pygeoprocessing.routing.distance_to_channel_mfd,
func=d_dn_func,
args=(
(f_reg['flow_direction_path'], 1),
(drainage_raster_path_task[0], 1),
@ -880,10 +918,13 @@ def execute(args):
sed_deposition_task = task_graph.add_task(
func=sdr_core.calculate_sediment_deposition,
args=(
f_reg['flow_direction_path'], f_reg['e_prime_path'],
f_reg['f_path'], f_reg['sdr_path'],
f_reg['sed_deposition_path']),
kwargs=dict(
flow_direction_path=f_reg['flow_direction_path'],
e_prime_path=f_reg['e_prime_path'],
f_path=f_reg['f_path'],
sdr_path=f_reg['sdr_path'],
target_sediment_deposition_path=f_reg['sed_deposition_path'],
algorithm=args['flow_dir_algorithm']),
dependent_task_list=[e_prime_task, sdr_task, flow_dir_task],
target_path_list=[f_reg['sed_deposition_path'], f_reg['f_path']],
task_name='sediment deposition')
@ -960,6 +1001,7 @@ def _avoided_export_op(avoided_erosion, sdr, sed_deposition):
# known as a modified USLE)
return avoided_erosion * sdr + sed_deposition
def add_drainage_op(stream, drainage):
"""raster_map equation: add drainage mask to stream layer.
@ -974,63 +1016,63 @@ def add_drainage_op(stream, drainage):
"""
return numpy.where(drainage == 1, 1, stream)
# raster_map equation: calculate USLE
def usle_op(rkls, cp_factor): return rkls * cp_factor
# raster_map equation: calculate the inverse ws factor
def inverse_ws_op(w_factor, s_factor): return 1 / (w_factor * s_factor)
def _calculate_what_drains_to_stream(
flow_dir_mfd_path, dist_to_channel_mfd_path, target_mask_path):
flow_dir_path, dist_to_channel_path, target_mask_path):
"""Create a mask indicating regions that do or do not drain to a stream.
This is useful because ``pygeoprocessing.distance_to_stream_mfd`` may leave
This is useful because the distance-to-stream functions may leave
some unexpected regions as nodata if they do not drain to a stream. This
may be confusing behavior, so this mask is intended to locate what drains
to a stream and what does not. A pixel doesn't drain to a stream if it has
a defined flow direction but undefined distance to stream.
Args:
flow_dir_mfd_path (string): The path to an MFD flow direction raster.
This raster must have a nodata value defined.
dist_to_channel_mfd_path (string): The path to an MFD
distance-to-channel raster. This raster must have a nodata value
defined.
flow_dir_path (string): The path to a flow direction raster
(MFD or D8). This raster must have a nodata value defined.
dist_to_channel_path (string): The path to a distance-to-channel
raster. This raster must have a nodata value defined.
target_mask_path (string): The path to where the mask raster should be
written.
Returns:
``None``
"""
flow_dir_mfd_nodata = pygeoprocessing.get_raster_info(
flow_dir_mfd_path)['nodata'][0]
flow_dir_nodata = pygeoprocessing.get_raster_info(
flow_dir_path)['nodata'][0]
dist_to_channel_nodata = pygeoprocessing.get_raster_info(
dist_to_channel_mfd_path)['nodata'][0]
dist_to_channel_path)['nodata'][0]
def _what_drains_to_stream(flow_dir_mfd, dist_to_channel):
def _what_drains_to_stream(flow_dir, dist_to_channel):
"""Determine which pixels do and do not drain to a stream.
Args:
flow_dir_mfd (numpy.array): A numpy array of MFD flow direction
values.
flow_dir (numpy.array): A numpy array of flow direction values.
dist_to_channel (numpy.array): A numpy array of calculated
distances to the nearest channel.
Returns:
A ``numpy.array`` of dtype ``numpy.uint8`` with pixels where:
* ``255`` where ``flow_dir_mfd`` is nodata (and thus
* ``255`` where ``flow_dir`` is nodata (and thus
``dist_to_channel`` is also nodata).
* ``0`` where ``flow_dir_mfd`` has data and ``dist_to_channel``
* ``0`` where ``flow_dir`` has data and ``dist_to_channel``
does not
* ``1`` where ``flow_dir_mfd`` has data, and
* ``1`` where ``flow_dir`` has data, and
``dist_to_channel`` also has data.
"""
drains_to_stream = numpy.full(
flow_dir_mfd.shape, _BYTE_NODATA, dtype=numpy.uint8)
flow_dir.shape, _BYTE_NODATA, dtype=numpy.uint8)
valid_flow_dir = ~pygeoprocessing.array_equals_nodata(
flow_dir_mfd, flow_dir_mfd_nodata)
flow_dir, flow_dir_nodata)
valid_dist_to_channel = (
~pygeoprocessing.array_equals_nodata(
dist_to_channel, dist_to_channel_nodata) &
@ -1044,7 +1086,7 @@ def _calculate_what_drains_to_stream(
return drains_to_stream
pygeoprocessing.raster_calculator(
[(flow_dir_mfd_path, 1), (dist_to_channel_mfd_path, 1)],
[(flow_dir_path, 1), (dist_to_channel_path, 1)],
_what_drains_to_stream, target_mask_path, gdal.GDT_Byte, _BYTE_NODATA)
@ -1185,7 +1227,7 @@ def _calculate_ls_factor(
def _calculate_rkls(
ls_factor_path, erosivity_path, erodibility_path, stream_path,
rkls_path):
"""Calculate potential soil loss (tons / (pixel * year)) using RKLS.
"""Calculate potential soil loss (tons / (ha * year)) using RKLS.
(revised universal soil loss equation with no C or P).
@ -1211,10 +1253,6 @@ def _calculate_rkls(
stream_nodata = pygeoprocessing.get_raster_info(
stream_path)['nodata'][0]
cell_size = abs(
pygeoprocessing.get_raster_info(ls_factor_path)['pixel_size'][0])
cell_area_ha = cell_size**2 / 10000.0 # hectares per pixel
def rkls_function(ls_factor, erosivity, erodibility, stream):
"""Calculate the RKLS equation.
@ -1227,7 +1265,7 @@ def _calculate_rkls(
stream (numpy.ndarray): stream mask (1 stream, 0 no stream)
Returns:
numpy.ndarray of RKLS values in tons / (pixel * year))
numpy.ndarray of RKLS values in tons / (ha * year))
"""
rkls = numpy.empty(ls_factor.shape, dtype=numpy.float32)
nodata_mask = (
@ -1243,11 +1281,10 @@ def _calculate_rkls(
valid_mask = nodata_mask & (stream == 0)
rkls[:] = _TARGET_NODATA
rkls[valid_mask] = ( # rkls units are tons / (pixel * year)
rkls[valid_mask] = ( # rkls units are tons / (ha * year)
ls_factor[valid_mask] * # unitless
erosivity[valid_mask] * # MJ * mm / (ha * hr * yr)
erodibility[valid_mask] * # t * ha * hr / (MJ * ha * mm)
cell_area_ha) # ha / pixel
erodibility[valid_mask]) # t * ha * hr / (MJ * ha * mm)
return rkls
# aligning with index 3 that's the stream and the most likely to be
@ -1338,7 +1375,7 @@ def _calculate_cp(lulc_to_cp, lulc_path, cp_factor_path):
def _calculate_bar_factor(
flow_direction_path, factor_path, flow_accumulation_path,
accumulation_path, out_bar_path):
accumulation_path, out_bar_path, flow_dir_algorithm):
"""Route user defined source across DEM.
Used for calculating S and W bar in the SDR operation.
@ -1353,15 +1390,21 @@ def _calculate_bar_factor(
out_bar_path (string): path to output raster that is the result of
the factor accumulation raster divided by the flow accumulation
raster.
flow_dir_algorithm (string): flow direction algorithm, 'D8' or 'MFD'
Returns:
None.
"""
LOGGER.debug("doing flow accumulation mfd on %s", factor_path)
LOGGER.debug(f"doing flow accumulation on {factor_path}")
if flow_dir_algorithm == 'D8':
flow_accum_func = pygeoprocessing.routing.flow_accumulation_d8
else: # MFD
flow_accum_func = pygeoprocessing.routing.flow_accumulation_mfd
# manually setting compression to DEFLATE because we got some LZW
# errors when testing with large data.
pygeoprocessing.routing.flow_accumulation_mfd(
flow_accum_func(
(flow_direction_path, 1), accumulation_path,
weight_raster_path_band=(factor_path, 1),
raster_driver_creation_tuple=('GTIFF', [
@ -1386,18 +1429,6 @@ def _calculate_d_up(
target_path=out_d_up_path)
def _calculate_d_up_bare(
s_bar_path, flow_accumulation_path, out_d_up_bare_path):
"""Calculate s_bar * sqrt(flow accumulation * cell area)."""
cell_area = abs(
pygeoprocessing.get_raster_info(s_bar_path)['pixel_size'][0])**2
pygeoprocessing.raster_map(
op=lambda s_bar, flow_accum: (
numpy.sqrt(flow_accum * cell_area) * s_bar),
rasters=[s_bar_path, flow_accumulation_path],
target_path=out_d_up_bare_path)
def _calculate_ic(d_up_path, d_dn_path, out_ic_factor_path):
"""Calculate log10(d_up/d_dn)."""
# ic can be positive or negative, so float.min is a reasonable nodata value
@ -1521,13 +1552,20 @@ def _generate_report(
field_def.SetPrecision(11)
target_layer.CreateField(field_def)
# Since pixel values are t/(ha•yr), raster sum is (t•px)/(ha•yr).
# To convert to t/yr, multiply by ha/px.
raster_info = pygeoprocessing.get_raster_info(usle_path)
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
ha_per_px = pixel_area / 10000
target_layer.ResetReading()
for feature in target_layer:
feature_id = feature.GetFID()
for field_name in field_summaries:
feature.SetField(
field_name,
float(field_summaries[field_name][feature_id]['sum']))
float(field_summaries[field_name][feature_id]['sum']
* ha_per_px))
target_layer.SetFeature(feature)
target_vector = None
target_layer = None

626
src/natcap/invest/sdr/sdr_core.pyx
View File

@ -1,357 +1,19 @@
import logging
import os
import numpy
import pygeoprocessing
cimport numpy
cimport cython
from osgeo import gdal
from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cython.operator cimport dereference as deref
from cython.operator cimport preincrement as inc
from libc.time cimport time as ctime
from libcpp.pair cimport pair
from libcpp.set cimport set as cset
from libcpp.list cimport list as clist
from libcpp.stack cimport stack
cimport libc.math as cmath
from ..managed_raster.managed_raster cimport D8
from ..managed_raster.managed_raster cimport MFD
from .sediment_deposition cimport run_sediment_deposition
cdef extern from "time.h" nogil:
ctypedef int time_t
time_t time(time_t*)
LOGGER = logging.getLogger(__name__)
# cmath is supposed to have M_SQRT2, but tests have been failing recently
# due to a missing symbol.
cdef double SQRT2 = cmath.sqrt(2)
cdef double PI = 3.141592653589793238462643383279502884
# This module creates rasters with a memory xy block size of 2**BLOCK_BITS
cdef int BLOCK_BITS = 8
# Number of raster blocks to hold in memory at once per Managed Raster
cdef int MANAGED_RASTER_N_BLOCKS = 2**6
# These offsets are for the neighbor rows and columns according to the
# ordering: 3 2 1
# 4 x 0
# 5 6 7
cdef int *ROW_OFFSETS = [0, -1, -1, -1, 0, 1, 1, 1]
cdef int *COL_OFFSETS = [1, 1, 0, -1, -1, -1, 0, 1]
# this is a least recently used cache written in C++ in an external file,
# exposing here so _ManagedRaster can use it
cdef extern from "LRUCache.h" nogil:
cdef cppclass LRUCache[KEY_T, VAL_T]:
LRUCache(int)
void put(KEY_T&, VAL_T&, clist[pair[KEY_T,VAL_T]]&)
clist[pair[KEY_T,VAL_T]].iterator begin()
clist[pair[KEY_T,VAL_T]].iterator end()
bint exist(KEY_T &)
VAL_T get(KEY_T &)
# this ctype is used to store the block ID and the block buffer as one object
# inside Managed Raster
ctypedef pair[int, double*] BlockBufferPair
# a class to allow fast random per-pixel access to a raster for both setting
# and reading pixels. Copied from src/pygeoprocessing/routing/routing.pyx,
# revision 891288683889237cfd3a3d0a1f09483c23489fca.
cdef class _ManagedRaster:
cdef LRUCache[int, double*]* lru_cache
cdef cset[int] dirty_blocks
cdef int block_xsize
cdef int block_ysize
cdef int block_xmod
cdef int block_ymod
cdef int block_xbits
cdef int block_ybits
cdef long raster_x_size
cdef long raster_y_size
cdef int block_nx
cdef int block_ny
cdef int write_mode
cdef bytes raster_path
cdef int band_id
cdef int closed
def __cinit__(self, raster_path, band_id, write_mode):
"""Create new instance of Managed Raster.
Args:
raster_path (char*): path to raster that has block sizes that are
powers of 2. If not, an exception is raised.
band_id (int): which band in `raster_path` to index. Uses GDAL
notation that starts at 1.
write_mode (boolean): if true, this raster is writable and dirty
memory blocks will be written back to the raster as blocks
are swapped out of the cache or when the object deconstructs.
Returns:
None.
"""
raster_info = pygeoprocessing.get_raster_info(raster_path)
self.raster_x_size, self.raster_y_size = raster_info['raster_size']
self.block_xsize, self.block_ysize = raster_info['block_size']
self.block_xmod = self.block_xsize-1
self.block_ymod = self.block_ysize-1
if not (1 <= band_id <= raster_info['n_bands']):
err_msg = (
"Error: band ID (%s) is not a valid band number. "
"This exception is happening in Cython, so it will cause a "
"hard seg-fault, but it's otherwise meant to be a "
"ValueError." % (band_id))
print(err_msg)
raise ValueError(err_msg)
self.band_id = band_id
if (self.block_xsize & (self.block_xsize - 1) != 0) or (
self.block_ysize & (self.block_ysize - 1) != 0):
# If inputs are not a power of two, this will at least print
# an error message. Unfortunately with Cython, the exception will
# present itself as a hard seg-fault, but I'm leaving the
# ValueError in here at least for readability.
err_msg = (
"Error: Block size is not a power of two: "
"block_xsize: %d, %d, %s. This exception is happening"
"in Cython, so it will cause a hard seg-fault, but it's"
"otherwise meant to be a ValueError." % (
self.block_xsize, self.block_ysize, raster_path))
print(err_msg)
raise ValueError(err_msg)
self.block_xbits = numpy.log2(self.block_xsize)
self.block_ybits = numpy.log2(self.block_ysize)
self.block_nx = (
self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize
self.block_ny = (
self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize
self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS)
self.raster_path = <bytes> raster_path
self.write_mode = write_mode
self.closed = 0
def __dealloc__(self):
"""Deallocate _ManagedRaster.
This operation manually frees memory from the LRUCache and writes any
dirty memory blocks back to the raster if `self.write_mode` is True.
"""
self.close()
def close(self):
"""Close the _ManagedRaster and free up resources.
This call writes any dirty blocks to disk, frees up the memory
allocated as part of the cache, and frees all GDAL references.
Any subsequent calls to any other functions in _ManagedRaster will
have undefined behavior.
"""
if self.closed:
return
self.closed = 1
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array = numpy.empty(
(self.block_ysize, self.block_xsize))
cdef double *double_buffer
cdef int block_xi
cdef int block_yi
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize
cdef int win_ysize
# we need the offsets to subtract from global indexes for cached array
cdef int xoff
cdef int yoff
cdef clist[BlockBufferPair].iterator it = self.lru_cache.begin()
cdef clist[BlockBufferPair].iterator end = self.lru_cache.end()
if not self.write_mode:
while it != end:
# write the changed value back if desired
PyMem_Free(deref(it).second)
inc(it)
return
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
# if we get here, we're in write_mode
cdef cset[int].iterator dirty_itr
while it != end:
double_buffer = deref(it).second
block_index = deref(it).first
# write to disk if block is dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for
# cached array
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
# clip window sizes if necessary
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = (
double_buffer[
(yi_copy << self.block_xbits) + xi_copy])
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
inc(it)
raster_band.FlushCache()
raster_band = None
raster = None
cdef inline void set(self, long xi, long yi, double value):
"""Set the pixel at `xi,yi` to `value`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))] = value
if self.write_mode:
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr == self.dirty_blocks.end():
self.dirty_blocks.insert(block_index)
cdef inline double get(self, long xi, long yi):
"""Return the value of the pixel at `xi,yi`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
return self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))]
cdef void _load_block(self, int block_index) except *:
cdef int block_xi = block_index % self.block_nx
cdef int block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for cached array
cdef int xoff = block_xi << self.block_xbits
cdef int yoff = block_yi << self.block_ybits
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array
cdef double *double_buffer
cdef clist[BlockBufferPair] removed_value_list
# determine the block aligned xoffset for read as array
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize = self.block_xsize
cdef int win_ysize = self.block_ysize
# load a new block
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (yoff+win_ysize - self.raster_y_size)
raster = gdal.OpenEx(self.raster_path, gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = raster_band.ReadAsArray(
xoff=xoff, yoff=yoff, win_xsize=win_xsize,
win_ysize=win_ysize).astype(
numpy.float64)
raster_band = None
raster = None
double_buffer = <double*>PyMem_Malloc(
(sizeof(double) << self.block_xbits) * win_ysize)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
double_buffer[(yi_copy<<self.block_xbits)+xi_copy] = (
block_array[yi_copy, xi_copy])
self.lru_cache.put(
<int>block_index, <double*>double_buffer, removed_value_list)
if self.write_mode:
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = numpy.empty(
(self.block_ysize, self.block_xsize), dtype=numpy.double)
while not removed_value_list.empty():
# write the changed value back if desired
double_buffer = removed_value_list.front().second
if self.write_mode:
block_index = removed_value_list.front().first
# write back the block if it's dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = double_buffer[
(yi_copy << self.block_xbits) + xi_copy]
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
removed_value_list.pop_front()
if self.write_mode:
raster_band = None
raster = None
def calculate_sediment_deposition(
mfd_flow_direction_path, e_prime_path, f_path, sdr_path,
target_sediment_deposition_path):
flow_direction_path, e_prime_path, f_path, sdr_path,
target_sediment_deposition_path, algorithm):
"""Calculate sediment deposition layer.
This algorithm outputs both sediment deposition (t_i) and flux (f_i)::
@ -389,8 +51,8 @@ def calculate_sediment_deposition(
will come from the SDR model and have nodata in the same places.
Args:
mfd_flow_direction_path (string): a path to a raster with
pygeoprocessing.routing MFD flow direction values.
flow_direction_path (string): a path to a flow direction raster,
in either MFD or D8 format. Specify with the ``algorithm`` arg.
e_prime_path (string): path to a raster that shows sources of
sediment that wash off a pixel but do not reach the stream.
f_path (string): path to a raster that shows the sediment flux
@ -398,6 +60,7 @@ def calculate_sediment_deposition(
sdr_path (string): path to Sediment Delivery Ratio raster.
target_sediment_deposition_path (string): path to created that
shows where the E' sources end up across the landscape.
algorithm (string): MFD or D8
Returns:
None.
@ -406,268 +69,19 @@ def calculate_sediment_deposition(
LOGGER.info('Calculate sediment deposition')
cdef float target_nodata = -1
pygeoprocessing.new_raster_from_base(
mfd_flow_direction_path, target_sediment_deposition_path,
flow_direction_path, target_sediment_deposition_path,
gdal.GDT_Float32, [target_nodata])
pygeoprocessing.new_raster_from_base(
mfd_flow_direction_path, f_path,
flow_direction_path, f_path,
gdal.GDT_Float32, [target_nodata])
cdef _ManagedRaster mfd_flow_direction_raster = _ManagedRaster(
mfd_flow_direction_path, 1, False)
cdef _ManagedRaster e_prime_raster = _ManagedRaster(
e_prime_path, 1, False)
cdef _ManagedRaster sdr_raster = _ManagedRaster(sdr_path, 1, False)
cdef _ManagedRaster f_raster = _ManagedRaster(f_path, 1, True)
cdef _ManagedRaster sediment_deposition_raster = _ManagedRaster(
target_sediment_deposition_path, 1, True)
# given the pixel neighbor numbering system
# 3 2 1
# 4 x 0
# 5 6 7
# if a pixel `x` has a neighbor `n` in position `i`,
# then `n`'s neighbor in position `inflow_offsets[i]`
# is the original pixel `x`
cdef int *inflow_offsets = [4, 5, 6, 7, 0, 1, 2, 3]
cdef long n_cols, n_rows
flow_dir_info = pygeoprocessing.get_raster_info(mfd_flow_direction_path)
n_cols, n_rows = flow_dir_info['raster_size']
cdef int mfd_nodata = 0
cdef stack[long] processing_stack
cdef float sdr_nodata = pygeoprocessing.get_raster_info(
sdr_path)['nodata'][0]
cdef float e_prime_nodata = pygeoprocessing.get_raster_info(
e_prime_path)['nodata'][0]
cdef long col_index, row_index, win_xsize, win_ysize, xoff, yoff
cdef long global_col, global_row, j, k
cdef long flat_index
cdef long seed_col = 0
cdef long seed_row = 0
cdef long neighbor_row, neighbor_col, ds_neighbor_row, ds_neighbor_col
cdef int flow_val, neighbor_flow_val, ds_neighbor_flow_val
cdef int flow_weight, neighbor_flow_weight
cdef float flow_sum, neighbor_flow_sum
cdef float downslope_sdr_weighted_sum, sdr_i, sdr_j
cdef float p_j, p_val
cdef unsigned long n_pixels_processed = 0
cdef time_t last_log_time = ctime(NULL)
for offset_dict in pygeoprocessing.iterblocks(
(mfd_flow_direction_path, 1), offset_only=True, largest_block=0):
win_xsize = offset_dict['win_xsize']
win_ysize = offset_dict['win_ysize']
xoff = offset_dict['xoff']
yoff = offset_dict['yoff']
if ctime(NULL) - last_log_time > 5.0:
last_log_time = ctime(NULL)
LOGGER.info('Sediment deposition %.2f%% complete', 100 * (
n_pixels_processed / float(n_cols * n_rows)))
for row_index in range(win_ysize):
seed_row = yoff + row_index
for col_index in range(win_xsize):
seed_col = xoff + col_index
# check if this is a good seed pixel ( a local high point)
if mfd_flow_direction_raster.get(seed_col, seed_row) == mfd_nodata:
continue
seed_pixel = 1
# iterate over each of the pixel's neighbors
for j in range(8):
# skip if the neighbor is outside the raster bounds
neighbor_row = seed_row + ROW_OFFSETS[j]
if neighbor_row < 0 or neighbor_row >= n_rows:
continue
neighbor_col = seed_col + COL_OFFSETS[j]
if neighbor_col < 0 or neighbor_col >= n_cols:
continue
# skip if the neighbor's flow direction is undefined
neighbor_flow_val = <int>mfd_flow_direction_raster.get(
neighbor_col, neighbor_row)
if neighbor_flow_val == mfd_nodata:
continue
# if the neighbor flows into it, it's not a local high
# point and so can't be a seed pixel
neighbor_flow_weight = (
neighbor_flow_val >> (inflow_offsets[j]*4)) & 0xF
if neighbor_flow_weight > 0:
seed_pixel = 0 # neighbor flows in, not a seed
break
# if this can be a seed pixel and hasn't already been
# calculated, put it on the stack
if seed_pixel and sediment_deposition_raster.get(
seed_col, seed_row) == target_nodata:
processing_stack.push(seed_row * n_cols + seed_col)
while processing_stack.size() > 0:
# loop invariant: cell has all upslope neighbors
# processed. this is true for seed pixels because they
# have no upslope neighbors.
flat_index = processing_stack.top()
processing_stack.pop()
global_row = flat_index // n_cols
global_col = flat_index % n_cols
# (sum over j ∈ J of f_j * p(i,j) in the equation for t_i)
# calculate the upslope f_j contribution to this pixel,
# the weighted sum of flux flowing onto this pixel from
# all neighbors
f_j_weighted_sum = 0
for j in range(8):
neighbor_row = global_row + ROW_OFFSETS[j]
if neighbor_row < 0 or neighbor_row >= n_rows:
continue
neighbor_col = global_col + COL_OFFSETS[j]
if neighbor_col < 0 or neighbor_col >= n_cols:
continue
# see if there's an inflow from the neighbor to the
# pixel
neighbor_flow_val = (
<int>mfd_flow_direction_raster.get(
neighbor_col, neighbor_row))
neighbor_flow_weight = (
neighbor_flow_val >> (inflow_offsets[j]*4)) & 0xF
if neighbor_flow_weight > 0:
f_j = f_raster.get(neighbor_col, neighbor_row)
if f_j == target_nodata:
continue
# sum up the neighbor's flow dir values in each
# direction.
# flow dir values are relative to the total
neighbor_flow_sum = 0
for k in range(8):
neighbor_flow_sum += (
neighbor_flow_val >> (k*4)) & 0xF
# get the proportion of the neighbor's flow that
# flows into the original pixel
p_val = neighbor_flow_weight / neighbor_flow_sum
# add the neighbor's flux value, weighted by the
# flow proportion
f_j_weighted_sum += p_val * f_j
# calculate sum of SDR values of immediate downslope
# neighbors, weighted by proportion of flow into each
# neighbor
# (sum over k ∈ K of SDR_k * p(i,k) in the equation above)
downslope_sdr_weighted_sum = 0
flow_val = <int>mfd_flow_direction_raster.get(
global_col, global_row)
flow_sum = 0
for k in range(8):
flow_sum += (flow_val >> (k*4)) & 0xF
# iterate over the neighbors again
for j in range(8):
# skip if neighbor is outside the raster boundaries
neighbor_row = global_row + ROW_OFFSETS[j]
if neighbor_row < 0 or neighbor_row >= n_rows:
continue
neighbor_col = global_col + COL_OFFSETS[j]
if neighbor_col < 0 or neighbor_col >= n_cols:
continue
# if it is a downslope neighbor, add to the sum and
# check if it can be pushed onto the stack yet
flow_weight = (flow_val >> (j*4)) & 0xF
if flow_weight > 0:
sdr_j = sdr_raster.get(neighbor_col, neighbor_row)
if sdr_j == sdr_nodata:
continue
if sdr_j == 0:
# this means it's a stream, for SDR deposition
# purposes, we set sdr to 1 to indicate this
# is the last step on which to retain sediment
sdr_j = 1
p_j = flow_weight / flow_sum
downslope_sdr_weighted_sum += sdr_j * p_j
# check if we can add neighbor j to the stack yet
#
# if there is a downslope neighbor it
# couldn't have been pushed on the processing
# stack yet, because the upslope was just
# completed
upslope_neighbors_processed = 1
# iterate over each neighbor-of-neighbor
for k in range(8):
# no need to push the one we're currently
# calculating back onto the stack
if inflow_offsets[k] == j:
continue
# skip if neighbor-of-neighbor is outside
# raster bounds
ds_neighbor_row = (
neighbor_row + ROW_OFFSETS[k])
if ds_neighbor_row < 0 or ds_neighbor_row >= n_rows:
continue
ds_neighbor_col = (
neighbor_col + COL_OFFSETS[k])
if ds_neighbor_col < 0 or ds_neighbor_col >= n_cols:
continue
# if any upslope neighbor of j hasn't been
# calculated, we can't push j onto the stack
# yet
ds_neighbor_flow_val = (
<int>mfd_flow_direction_raster.get(
ds_neighbor_col, ds_neighbor_row))
if (ds_neighbor_flow_val >> (
inflow_offsets[k]*4)) & 0xF > 0:
if (sediment_deposition_raster.get(
ds_neighbor_col, ds_neighbor_row) ==
target_nodata):
upslope_neighbors_processed = 0
break
# if all upslope neighbors of neighbor j are
# processed, we can push j onto the stack.
if upslope_neighbors_processed:
processing_stack.push(
neighbor_row * n_cols +
neighbor_col)
# nodata pixels should propagate to the results
sdr_i = sdr_raster.get(global_col, global_row)
if sdr_i == sdr_nodata:
continue
e_prime_i = e_prime_raster.get(global_col, global_row)
if e_prime_i == e_prime_nodata:
continue
# This condition reflects property A in the user's guide.
if downslope_sdr_weighted_sum < sdr_i:
# i think this happens because of our low resolution
# flow direction, it's okay to zero out.
downslope_sdr_weighted_sum = sdr_i
# these correspond to the full equations for
# dr_i, t_i, and f_i given in the docstring
if sdr_i == 1:
# This reflects property B in the user's guide and is
# an edge case to avoid division-by-zero.
dt_i = 1
else:
dt_i = (downslope_sdr_weighted_sum - sdr_i) / (1 - sdr_i)
# Lisa's modified equations
t_i = dt_i * f_j_weighted_sum # deposition, a.k.a trapped sediment
f_i = (1 - dt_i) * f_j_weighted_sum + e_prime_i # flux
# On large flow paths, it's possible for dt_i, f_i and t_i
# to have very small negative values that are numerically
# equivalent to 0. These negative values were raising
# questions on the forums and it's easier to clamp the
# values here than to explain IEEE 754.
if dt_i < 0:
dt_i = 0
if t_i < 0:
t_i = 0
if f_i < 0:
f_i = 0
sediment_deposition_raster.set(global_col, global_row, t_i)
f_raster.set(global_col, global_row, f_i)
n_pixels_processed += (win_xsize * win_ysize)
LOGGER.info('Sediment deposition 100% complete')
sediment_deposition_raster.close()
if algorithm == 'D8':
run_sediment_deposition[D8](
flow_direction_path.encode('utf-8'), e_prime_path.encode('utf-8'),
f_path.encode('utf-8'), sdr_path.encode('utf-8'),
target_sediment_deposition_path.encode('utf-8'))
else:
run_sediment_deposition[MFD](
flow_direction_path.encode('utf-8'), e_prime_path.encode('utf-8'),
f_path.encode('utf-8'), sdr_path.encode('utf-8'),
target_sediment_deposition_path.encode('utf-8'))

273
src/natcap/invest/sdr/sediment_deposition.h Normal file
View File

@ -0,0 +1,273 @@
#include "ManagedRaster.h"
#include <ctime>
// Calculate sediment deposition layer.
//
// This algorithm outputs both sediment deposition (t_i) and flux (f_i)::
//
// t_i = dt_i * (sum over j ∈ J of f_j * p(j,i))
//
// f_i = (1 - dt_i) * (sum over j ∈ J of f_j * p(j,i)) + E'_i
//
//
// (sum over k ∈ K of SDR_k * p(i,k)) - SDR_i
// dt_i = --------------------------------------------
// (1 - SDR_i)
//
// where:
//
// - ``p(i,j)`` is the proportion of flow from pixel ``i`` into pixel ``j``
// - ``J`` is the set of pixels that are immediate upslope neighbors of
// pixel ``i``
// - ``K`` is the set of pixels that are immediate downslope neighbors of
// pixel ``i``
// - ``E'`` is ``USLE * (1 - SDR)``, the amount of sediment loss from pixel
// ``i`` that doesn't reach a stream (``e_prime_path``)
// - ``SDR`` is the sediment delivery ratio (``sdr_path``)
//
// ``f_i`` is recursively defined in terms of ``i``'s upslope neighbors.
// The algorithm begins from seed pixels that are local high points and so
// have no upslope neighbors. It works downslope from each seed pixel,
// only adding a pixel to the stack when all its upslope neighbors are
// already calculated.
//
// Note that this function is designed to be used in the context of the SDR
// model. Because the algorithm is recursive upslope and downslope of each
// pixel, nodata values in the SDR input would propagate along the flow path.
// This case is not handled because we assume the SDR and flow dir inputs
// will come from the SDR model and have nodata in the same places.
//
// Args:
// flow_direction_path: a path to a flow direction raster,
// in either MFD or D8 format. Specify with the ``algorithm`` arg.
// e_prime_path: path to a raster that shows sources of
// sediment that wash off a pixel but do not reach the stream.
// f_path: path to a raster that shows the sediment flux
// on a pixel for sediment that does not reach the stream.
// sdr_path: path to Sediment Delivery Ratio raster.
// target_sediment_deposition_path: path to created that
// shows where the E' sources end up across the landscape.
template<class T>
void run_sediment_deposition(
char* flow_direction_path,
char* e_prime_path,
char* f_path,
char* sdr_path,
char* sediment_deposition_path) {
ManagedFlowDirRaster flow_dir_raster = ManagedFlowDirRaster<T>(
flow_direction_path, 1, false);
ManagedRaster e_prime_raster = ManagedRaster(e_prime_path, 1, false);
ManagedRaster sdr_raster = ManagedRaster(sdr_path, 1, false);
ManagedRaster f_raster = ManagedRaster(f_path, 1, true);
ManagedRaster sediment_deposition_raster = ManagedRaster(
sediment_deposition_path, 1, true);
stack<long> processing_stack;
float target_nodata = -1;
long win_xsize, win_ysize, xoff, yoff;
long global_col, global_row;
int xs, ys;
long flat_index;
double downslope_sdr_weighted_sum;
double sdr_i, e_prime_i, sdr_j, f_j;
long flow_dir_sum;
time_t last_log_time = time(NULL);
unsigned long n_pixels_processed = 0;
bool upslope_neighbors_processed;
double f_j_weighted_sum;
NeighborTuple neighbor;
NeighborTuple neighbor_of_neighbor;
double dr_i, t_i, f_i;
UpslopeNeighbors<T> up_neighbors;
DownslopeNeighbors<T> dn_neighbors;
float total_n_pixels = flow_dir_raster.raster_x_size * flow_dir_raster.raster_y_size;
// efficient way to calculate ceiling division:
// a divided by b rounded up = (a + (b - 1)) / b
// note that / represents integer floor division
// https://stackoverflow.com/a/62032709/14451410
int n_col_blocks = (flow_dir_raster.raster_x_size + (flow_dir_raster.block_xsize - 1)) / flow_dir_raster.block_xsize;
int n_row_blocks = (flow_dir_raster.raster_y_size + (flow_dir_raster.block_ysize - 1)) / flow_dir_raster.block_ysize;
for (int row_block_index = 0; row_block_index < n_row_blocks; row_block_index++) {
yoff = row_block_index * flow_dir_raster.block_ysize;
win_ysize = flow_dir_raster.raster_y_size - yoff;
if (win_ysize > flow_dir_raster.block_ysize) {
win_ysize = flow_dir_raster.block_ysize;
}
for (int col_block_index = 0; col_block_index < n_col_blocks; col_block_index++) {
xoff = col_block_index * flow_dir_raster.block_xsize;
win_xsize = flow_dir_raster.raster_x_size - xoff;
if (win_xsize > flow_dir_raster.block_xsize) {
win_xsize = flow_dir_raster.block_xsize;
}
if (time(NULL) - last_log_time > 5) {
last_log_time = time(NULL);
log_msg(
LogLevel::info,
"Sediment deposition " + std::to_string(
100 * n_pixels_processed / total_n_pixels
) + " complete"
);
}
for (int row_index = 0; row_index < win_ysize; row_index++) {
ys = yoff + row_index;
for (int col_index = 0; col_index < win_xsize; col_index++) {
xs = xoff + col_index;
if (flow_dir_raster.get(xs, ys) == flow_dir_raster.nodata) {
continue;
}
// if this can be a seed pixel and hasn't already been
// calculated, put it on the stack
if (flow_dir_raster.is_local_high_point(xs, ys) and
is_close(sediment_deposition_raster.get(xs, ys), target_nodata)) {
processing_stack.push(ys * flow_dir_raster.raster_x_size + xs);
}
while (processing_stack.size() > 0) {
// # loop invariant: cell has all upslope neighbors
// # processed. this is true for seed pixels because they
// # have no upslope neighbors.
flat_index = processing_stack.top();
processing_stack.pop();
global_row = flat_index / flow_dir_raster.raster_x_size;
global_col = flat_index % flow_dir_raster.raster_x_size;
// # (sum over j ∈ J of f_j * p(i,j) in the equation for t_i)
// # calculate the upslope f_j contribution to this pixel,
// # the weighted sum of flux flowing onto this pixel from
// # all neighbors
f_j_weighted_sum = 0;
up_neighbors = UpslopeNeighbors<T>(
Pixel<T>(flow_dir_raster, global_col, global_row));
for (auto neighbor: up_neighbors) {
f_j = f_raster.get(neighbor.x, neighbor.y);
if (is_close(f_j, target_nodata)) {
continue;
}
// add the neighbor's flux value, weighted by the
// flow proportion
f_j_weighted_sum += neighbor.flow_proportion * f_j;
}
// # calculate sum of SDR values of immediate downslope
// # neighbors, weighted by proportion of flow into each
// # neighbor
// # (sum over k ∈ K of SDR_k * p(i,k) in the equation above)
downslope_sdr_weighted_sum = 0;
dn_neighbors = DownslopeNeighbors<T>(
Pixel<T>(flow_dir_raster, global_col, global_row));
flow_dir_sum = 0;
for (auto neighbor: dn_neighbors) {
flow_dir_sum += static_cast<long>(neighbor.flow_proportion);
sdr_j = sdr_raster.get(neighbor.x, neighbor.y);
if (is_close(sdr_j, sdr_raster.nodata)) {
continue;
}
if (sdr_j == 0) {
// # this means it's a stream, for SDR deposition
// # purposes, we set sdr to 1 to indicate this
// # is the last step on which to retain sediment
sdr_j = 1;
}
downslope_sdr_weighted_sum += (sdr_j * neighbor.flow_proportion);
// # check if we can add neighbor j to the stack yet
// #
// # if there is a downslope neighbor it
// # couldn't have been pushed on the processing
// # stack yet, because the upslope was just
// # completed
upslope_neighbors_processed = true;
// # iterate over each neighbor-of-neighbor
up_neighbors = UpslopeNeighbors<T>(
Pixel<T>(flow_dir_raster, neighbor.x, neighbor.y));
for (auto neighbor_of_neighbor: up_neighbors) {
if (INFLOW_OFFSETS[neighbor_of_neighbor.direction] == neighbor.direction) {
continue;
}
if (is_close(sediment_deposition_raster.get(
neighbor_of_neighbor.x, neighbor_of_neighbor.y
), target_nodata)) {
upslope_neighbors_processed = false;
break;
}
}
// # if all upslope neighbors of neighbor j are
// # processed, we can push j onto the stack.
if (upslope_neighbors_processed) {
processing_stack.push(
neighbor.y * flow_dir_raster.raster_x_size + neighbor.x);
}
}
// # nodata pixels should propagate to the results
sdr_i = sdr_raster.get(global_col, global_row);
if (is_close(sdr_i, sdr_raster.nodata)) {
continue;
}
e_prime_i = e_prime_raster.get(global_col, global_row);
if (is_close(e_prime_i, e_prime_raster.nodata)) {
continue;
}
if (flow_dir_sum) {
downslope_sdr_weighted_sum /= flow_dir_sum;
}
// # This condition reflects property A in the user's guide.
if (downslope_sdr_weighted_sum < sdr_i) {
// # i think this happens because of our low resolution
// # flow direction, it's okay to zero out.
downslope_sdr_weighted_sum = sdr_i;
}
// # these correspond to the full equations for
// # dr_i, t_i, and f_i given in the docstring
if (sdr_i == 1) {
// # This reflects property B in the user's guide and is
// # an edge case to avoid division-by-zero.
dr_i = 1;
} else {
dr_i = (downslope_sdr_weighted_sum - sdr_i) / (1 - sdr_i);
}
// # Lisa's modified equations
t_i = dr_i * f_j_weighted_sum; // deposition, a.k.a trapped sediment
f_i = (1 - dr_i) * f_j_weighted_sum + e_prime_i; // flux
// # On large flow paths, it's possible for dr_i, f_i and t_i
// # to have very small negative values that are numerically
// # equivalent to 0. These negative values were raising
// # questions on the forums and it's easier to clamp the
// # values here than to explain IEEE 754.
if (dr_i < 0) {
dr_i = 0;
}
if (t_i < 0) {
t_i = 0;
}
if (f_i < 0) {
f_i = 0;
}
sediment_deposition_raster.set(global_col, global_row, t_i);
f_raster.set(global_col, global_row, f_i);
}
}
}
n_pixels_processed += win_xsize * win_ysize;
}
}
sediment_deposition_raster.close();
flow_dir_raster.close();
e_prime_raster.close();
sdr_raster.close();
f_raster.close();
log_msg(LogLevel::info, "Sediment deposition 100% complete");
}

7
src/natcap/invest/sdr/sediment_deposition.pxd Normal file
View File

@ -0,0 +1,7 @@
cdef extern from "sediment_deposition.h":
void run_sediment_deposition[T](
char*,
char*,
char*,
char*,
char*) except +

73
src/natcap/invest/seasonal_water_yield/LRUCache.h
View File

@ -1,73 +0,0 @@
#ifndef __LRUCACHE_H_INCLUDED__
#define __LRUCACHE_H_INCLUDED__
#include <list>
#include <map>
#include <assert.h>
using namespace std;
template <class KEY_T, class VAL_T,
typename ListIter = typename list< pair<KEY_T,VAL_T> >::iterator,
typename MapIter = typename map<KEY_T, ListIter>::iterator > class LRUCache{
private:
// item_list keeps track of the order of which elements have been accessed
// element at begin is most recent, element at end is least recent.
// first element in the pair is its key while the second is the element
list< pair<KEY_T,VAL_T> > item_list;
// item_map maps an element's key to its location in the `item_list`
// used to make lookups O(log n) time
map<KEY_T, ListIter> item_map;
size_t cache_size;
private:
void clean(list< pair<KEY_T, VAL_T> > &removed_value_list){
while(item_map.size()>cache_size){
ListIter last_it = item_list.end(); last_it --;
removed_value_list.push_back(
make_pair(last_it->first, last_it->second));
item_map.erase(last_it->first);
item_list.pop_back();
}
};
public:
LRUCache(int cache_size_):cache_size(cache_size_){
;
};
ListIter begin() {
return item_list.begin();
}
ListIter end() {
return item_list.end();
}
void put(
const KEY_T &key, const VAL_T &val,
list< pair<KEY_T, VAL_T> > &removed_value_list) {
MapIter it = item_map.find(key);
if(it != item_map.end()){
// it's already in the cache, delete the location in the item
// list and in the lookup map
item_list.erase(it->second);
item_map.erase(it);
}
// insert a new item in the front since it's most recently used
item_list.push_front(make_pair(key,val));
// record its iterator in the map
item_map.insert(make_pair(key, item_list.begin()));
// possibly remove any elements that have exceeded the cache size
return clean(removed_value_list);
};
bool exist(const KEY_T &key){
return (item_map.count(key)>0);
};
VAL_T& get(const KEY_T &key){
MapIter it = item_map.find(key);
assert(it!=item_map.end());
// move the element to the front of the list
item_list.splice(item_list.begin(), item_list, it->second);
return it->second->second;
};
};
#endif

141
src/natcap/invest/seasonal_water_yield/seasonal_water_yield.py
View File

@ -30,6 +30,7 @@ MONTH_ID_TO_LABEL = [
'nov', 'dec']
MODEL_SPEC = {
"model_id": "seasonal_water_yield",
"model_name": MODEL_METADATA["seasonal_water_yield"].model_title,
"pyname": MODEL_METADATA["seasonal_water_yield"].pyname,
"userguide": MODEL_METADATA["seasonal_water_yield"].userguide,
@ -292,7 +293,8 @@ MODEL_SPEC = {
"Table of alpha values for each month. "
"Required if Use Monthly Alpha Table is selected."),
"name": gettext("monthly alpha table")
}
},
**spec_utils.FLOW_DIR_ALGORITHM
},
"outputs": {
"B.tif": {
@ -407,10 +409,10 @@ MODEL_SPEC = {
"units": u.millimeter
}}
},
"flow_dir_mfd.tif": {
"flow_dir.tif": {
"about": gettext(
"Map of multiple flow direction. Values are encoded in "
"a binary format and should not be used directly."),
"Map of flow direction, in either D8 or MFD format "
"according to the option selected."),
"bands": {1: {"type": "integer"}}
},
"qf_[MONTH].tif": {
@ -506,7 +508,7 @@ _OUTPUT_BASE_FILES = {
_INTERMEDIATE_BASE_FILES = {
'aet_path': 'aet.tif',
'aetm_path_list': ['aetm_%d.tif' % (x+1) for x in range(N_MONTHS)],
'flow_dir_mfd_path': 'flow_dir_mfd.tif',
'flow_dir_path': 'flow_dir.tif',
'qfm_path_list': ['qf_%d.tif' % (x+1) for x in range(N_MONTHS)],
'stream_path': 'stream.tif',
'si_path': 'Si.tif',
@ -710,35 +712,67 @@ def execute(args):
dependent_task_list=[align_task],
task_name='fill dem pits')
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(file_registry['dem_pit_filled_path'], 1),
file_registry['flow_dir_mfd_path']),
kwargs={'working_dir': intermediate_output_dir},
target_path_list=[file_registry['flow_dir_mfd_path']],
dependent_task_list=[fill_pit_task],
task_name='flow dir mfd')
if args['flow_dir_algorithm'] == 'MFD':
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_mfd,
args=(
(file_registry['dem_pit_filled_path'], 1),
file_registry['flow_dir_path']),
kwargs={'working_dir': intermediate_output_dir},
target_path_list=[file_registry['flow_dir_path']],
dependent_task_list=[fill_pit_task],
task_name='flow direction - MFD')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(file_registry['flow_dir_path'], 1),
file_registry['flow_accum_path']),
target_path_list=[file_registry['flow_accum_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accumulation - MFD')
stream_threshold_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(file_registry['flow_accum_path'], 1),
(file_registry['flow_dir_path'], 1),
threshold_flow_accumulation,
file_registry['stream_path']),
target_path_list=[file_registry['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream threshold - MFD')
else: # D8
flow_dir_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_dir_d8,
args=(
(file_registry['dem_pit_filled_path'], 1),
file_registry['flow_dir_path']),
kwargs={'working_dir': intermediate_output_dir},
target_path_list=[file_registry['flow_dir_path']],
dependent_task_list=[fill_pit_task],
task_name='flow direction - D8')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_d8,
args=(
(file_registry['flow_dir_path'], 1),
file_registry['flow_accum_path']),
target_path_list=[file_registry['flow_accum_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accumulation - D8')
stream_threshold_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_d8,
kwargs=dict(
flow_accum_raster_path_band=(file_registry['flow_accum_path'], 1),
flow_threshold=threshold_flow_accumulation,
target_stream_raster_path=file_registry['stream_path']),
target_path_list=[file_registry['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream threshold - D8')
flow_accum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(file_registry['flow_dir_mfd_path'], 1),
file_registry['flow_accum_path']),
target_path_list=[file_registry['flow_accum_path']],
dependent_task_list=[flow_dir_task],
task_name='flow accum task')
stream_threshold_task = task_graph.add_task(
func=pygeoprocessing.routing.extract_streams_mfd,
args=(
(file_registry['flow_accum_path'], 1),
(file_registry['flow_dir_mfd_path'], 1),
threshold_flow_accumulation,
file_registry['stream_path']),
target_path_list=[file_registry['stream_path']],
dependent_task_list=[flow_accum_task],
task_name='stream threshold')
LOGGER.info('quick flow')
if args['user_defined_local_recharge']:
@ -871,7 +905,7 @@ def execute(args):
file_registry['precip_path_aligned_list'],
file_registry['et0_path_aligned_list'],
file_registry['qfm_path_list'],
file_registry['flow_dir_mfd_path'],
file_registry['flow_dir_path'],
file_registry['kc_path_list'],
alpha_month_map,
beta_i, gamma, file_registry['stream_path'],
@ -879,7 +913,8 @@ def execute(args):
file_registry['l_avail_path'],
file_registry['l_sum_avail_path'],
file_registry['aet_path'],
file_registry['annual_precip_path']),
file_registry['annual_precip_path'],
args['flow_dir_algorithm']),
target_path_list=[
file_registry['l_path'],
file_registry['l_avail_path'],
@ -916,16 +951,28 @@ def execute(args):
task_name='aggregate recharge')
LOGGER.info('calculate L_sum') # Eq. [12]
l_sum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(file_registry['flow_dir_mfd_path'], 1),
file_registry['l_sum_path']),
kwargs={'weight_raster_path_band': (file_registry['l_path'], 1)},
target_path_list=[file_registry['l_sum_path']],
dependent_task_list=vri_dependent_task_list + [
fill_pit_task, flow_dir_task, stream_threshold_task],
task_name='calculate l sum')
if args['flow_dir_algorithm'] == 'MFD':
l_sum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_mfd,
args=(
(file_registry['flow_dir_path'], 1),
file_registry['l_sum_path']),
kwargs={'weight_raster_path_band': (file_registry['l_path'], 1)},
target_path_list=[file_registry['l_sum_path']],
dependent_task_list=vri_dependent_task_list + [
fill_pit_task, flow_dir_task, stream_threshold_task],
task_name='calculate l sum - MFD')
else: # D8
l_sum_task = task_graph.add_task(
func=pygeoprocessing.routing.flow_accumulation_d8,
args=(
(file_registry['flow_dir_path'], 1),
file_registry['l_sum_path']),
kwargs={'weight_raster_path_band': (file_registry['l_path'], 1)},
target_path_list=[file_registry['l_sum_path']],
dependent_task_list=vri_dependent_task_list + [
fill_pit_task, flow_dir_task, stream_threshold_task],
task_name='calculate l sum - D8')
if args['user_defined_local_recharge']:
b_sum_dependent_task_list = [l_avail_task]
@ -935,14 +982,14 @@ def execute(args):
b_sum_task = task_graph.add_task(
func=seasonal_water_yield_core.route_baseflow_sum,
args=(
file_registry['flow_dir_mfd_path'],
file_registry['flow_dir_path'],
file_registry['l_path'],
file_registry['l_avail_path'],
file_registry['l_sum_path'],
file_registry['stream_path'],
file_registry['b_path'],
file_registry['b_sum_path']),
file_registry['b_sum_path'],
args['flow_dir_algorithm']),
target_path_list=[
file_registry['b_sum_path'], file_registry['b_path']],
dependent_task_list=b_sum_dependent_task_list + [l_sum_task],

834
src/natcap/invest/seasonal_water_yield/seasonal_water_yield_core.pyx
View File

@ -11,370 +11,19 @@ cimport cython
from osgeo import gdal
from osgeo import ogr
from osgeo import osr
from cython.operator cimport dereference as deref
from cpython.mem cimport PyMem_Malloc, PyMem_Free
from cython.operator cimport dereference as deref
from cython.operator cimport preincrement as inc
from libc.math cimport isnan
from libcpp.list cimport list as clist
from libcpp.set cimport set as cset
from libcpp.pair cimport pair
from libcpp.stack cimport stack
from libcpp.queue cimport queue
from libc.time cimport time as ctime
cdef extern from "time.h" nogil:
ctypedef int time_t
time_t time(time_t*)
cdef int is_close(double x, double y):
if isnan(x) and isnan(y):
return 1
return abs(x-y) <= (1e-8+1e-05*abs(y))
cdef extern from "LRUCache.h":
cdef cppclass LRUCache[KEY_T, VAL_T]:
LRUCache(int)
void put(KEY_T&, VAL_T&, clist[pair[KEY_T,VAL_T]]&)
clist[pair[KEY_T,VAL_T]].iterator begin()
clist[pair[KEY_T,VAL_T]].iterator end()
bint exist(KEY_T &)
VAL_T get(KEY_T &)
from libcpp.vector cimport vector
from ..managed_raster.managed_raster cimport D8, MFD
from .swy cimport run_route_baseflow_sum, run_calculate_local_recharge
LOGGER = logging.getLogger(__name__)
cdef int N_MONTHS = 12
cdef double PI = 3.141592653589793238462643383279502884
cdef double INF = numpy.inf
cdef double IMPROBABLE_FLOAT_NOATA = -1.23789789e29
# used to loop over neighbors and offset the x/y values as defined below
# 321
# 4x0
# 567
cdef int* NEIGHBOR_OFFSET_ARRAY = [
1, 0, # 0
1, -1, # 1
0, -1, # 2
-1, -1, # 3
-1, 0, # 4
-1, 1, # 5
0, 1, # 6
1, 1 # 7
]
# index into this array with a direction and get the index for the reverse
# direction. Useful for determining the direction a neighbor flows into a
# cell.
cdef int* FLOW_DIR_REVERSE_DIRECTION = [4, 5, 6, 7, 0, 1, 2, 3]
# this ctype is used to store the block ID and the block buffer as one object
# inside Managed Raster
ctypedef pair[int, double*] BlockBufferPair
# Number of raster blocks to hold in memory at once per Managed Raster
cdef int MANAGED_RASTER_N_BLOCKS = 2**4
# a class to allow fast random per-pixel access to a raster for both setting
# and reading pixels. Copied from src/pygeoprocessing/routing/routing.pyx,
# revision 891288683889237cfd3a3d0a1f09483c23489fca.
cdef class _ManagedRaster:
cdef LRUCache[int, double*]* lru_cache
cdef cset[int] dirty_blocks
cdef int block_xsize
cdef int block_ysize
cdef int block_xmod
cdef int block_ymod
cdef int block_xbits
cdef int block_ybits
cdef long raster_x_size
cdef long raster_y_size
cdef int block_nx
cdef int block_ny
cdef int write_mode
cdef bytes raster_path
cdef int band_id
cdef int closed
def __cinit__(self, raster_path, band_id, write_mode):
"""Create new instance of Managed Raster.
Args:
raster_path (char*): path to raster that has block sizes that are
powers of 2. If not, an exception is raised.
band_id (int): which band in `raster_path` to index. Uses GDAL
notation that starts at 1.
write_mode (boolean): if true, this raster is writable and dirty
memory blocks will be written back to the raster as blocks
are swapped out of the cache or when the object deconstructs.
Returns:
None.
"""
raster_info = pygeoprocessing.get_raster_info(raster_path)
self.raster_x_size, self.raster_y_size = raster_info['raster_size']
self.block_xsize, self.block_ysize = raster_info['block_size']
self.block_xmod = self.block_xsize-1
self.block_ymod = self.block_ysize-1
if not (1 <= band_id <= raster_info['n_bands']):
err_msg = (
"Error: band ID (%s) is not a valid band number. "
"This exception is happening in Cython, so it will cause a "
"hard seg-fault, but it's otherwise meant to be a "
"ValueError." % (band_id))
print(err_msg)
raise ValueError(err_msg)
self.band_id = band_id
if (self.block_xsize & (self.block_xsize - 1) != 0) or (
self.block_ysize & (self.block_ysize - 1) != 0):
# If inputs are not a power of two, this will at least print
# an error message. Unfortunately with Cython, the exception will
# present itself as a hard seg-fault, but I'm leaving the
# ValueError in here at least for readability.
err_msg = (
"Error: Block size is not a power of two: "
"block_xsize: %d, %d, %s. This exception is happening"
"in Cython, so it will cause a hard seg-fault, but it's"
"otherwise meant to be a ValueError." % (
self.block_xsize, self.block_ysize, raster_path))
print(err_msg)
raise ValueError(err_msg)
self.block_xbits = numpy.log2(self.block_xsize)
self.block_ybits = numpy.log2(self.block_ysize)
self.block_nx = (
self.raster_x_size + (self.block_xsize) - 1) // self.block_xsize
self.block_ny = (
self.raster_y_size + (self.block_ysize) - 1) // self.block_ysize
self.lru_cache = new LRUCache[int, double*](MANAGED_RASTER_N_BLOCKS)
self.raster_path = <bytes> raster_path
self.write_mode = write_mode
self.closed = 0
def __dealloc__(self):
"""Deallocate _ManagedRaster.
This operation manually frees memory from the LRUCache and writes any
dirty memory blocks back to the raster if `self.write_mode` is True.
"""
self.close()
def close(self):
"""Close the _ManagedRaster and free up resources.
This call writes any dirty blocks to disk, frees up the memory
allocated as part of the cache, and frees all GDAL references.
Any subsequent calls to any other functions in _ManagedRaster will
have undefined behavior.
"""
if self.closed:
return
self.closed = 1
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array = numpy.empty(
(self.block_ysize, self.block_xsize))
cdef double *double_buffer
cdef int block_xi
cdef int block_yi
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize
cdef int win_ysize
# we need the offsets to subtract from global indexes for cached array
cdef int xoff
cdef int yoff
cdef clist[BlockBufferPair].iterator it = self.lru_cache.begin()
cdef clist[BlockBufferPair].iterator end = self.lru_cache.end()
if not self.write_mode:
while it != end:
# write the changed value back if desired
PyMem_Free(deref(it).second)
inc(it)
return
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
# if we get here, we're in write_mode
cdef cset[int].iterator dirty_itr
while it != end:
double_buffer = deref(it).second
block_index = deref(it).first
# write to disk if block is dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for
# cached array
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
# clip window sizes if necessary
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = (
double_buffer[
(yi_copy << self.block_xbits) + xi_copy])
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
inc(it)
raster_band.FlushCache()
raster_band = None
raster = None
cdef inline void set(self, long xi, long yi, double value):
"""Set the pixel at `xi,yi` to `value`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))] = value
if self.write_mode:
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr == self.dirty_blocks.end():
self.dirty_blocks.insert(block_index)
cdef inline double get(self, long xi, long yi):
"""Return the value of the pixel at `xi,yi`."""
cdef int block_xi = xi >> self.block_xbits
cdef int block_yi = yi >> self.block_ybits
# this is the flat index for the block
cdef int block_index = block_yi * self.block_nx + block_xi
if not self.lru_cache.exist(block_index):
self._load_block(block_index)
return self.lru_cache.get(
block_index)[
((yi & (self.block_ymod))<<self.block_xbits) +
(xi & (self.block_xmod))]
cdef void _load_block(self, int block_index) except *:
cdef int block_xi = block_index % self.block_nx
cdef int block_yi = block_index // self.block_nx
# we need the offsets to subtract from global indexes for cached array
cdef int xoff = block_xi << self.block_xbits
cdef int yoff = block_yi << self.block_ybits
cdef int xi_copy, yi_copy
cdef numpy.ndarray[double, ndim=2] block_array
cdef double *double_buffer
cdef clist[BlockBufferPair] removed_value_list
# determine the block aligned xoffset for read as array
# initially the win size is the same as the block size unless
# we're at the edge of a raster
cdef int win_xsize = self.block_xsize
cdef int win_ysize = self.block_ysize
# load a new block
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (yoff+win_ysize - self.raster_y_size)
raster = gdal.OpenEx(self.raster_path, gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = raster_band.ReadAsArray(
xoff=xoff, yoff=yoff, win_xsize=win_xsize,
win_ysize=win_ysize).astype(
numpy.float64)
raster_band = None
raster = None
double_buffer = <double*>PyMem_Malloc(
(sizeof(double) << self.block_xbits) * win_ysize)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
double_buffer[(yi_copy<<self.block_xbits)+xi_copy] = (
block_array[yi_copy, xi_copy])
self.lru_cache.put(
<int>block_index, <double*>double_buffer, removed_value_list)
if self.write_mode:
raster = gdal.OpenEx(
self.raster_path, gdal.GA_Update | gdal.OF_RASTER)
raster_band = raster.GetRasterBand(self.band_id)
block_array = numpy.empty(
(self.block_ysize, self.block_xsize), dtype=numpy.double)
while not removed_value_list.empty():
# write the changed value back if desired
double_buffer = removed_value_list.front().second
if self.write_mode:
block_index = removed_value_list.front().first
# write back the block if it's dirty
dirty_itr = self.dirty_blocks.find(block_index)
if dirty_itr != self.dirty_blocks.end():
self.dirty_blocks.erase(dirty_itr)
block_xi = block_index % self.block_nx
block_yi = block_index // self.block_nx
xoff = block_xi << self.block_xbits
yoff = block_yi << self.block_ybits
win_xsize = self.block_xsize
win_ysize = self.block_ysize
if xoff+win_xsize > self.raster_x_size:
win_xsize = win_xsize - (
xoff+win_xsize - self.raster_x_size)
if yoff+win_ysize > self.raster_y_size:
win_ysize = win_ysize - (
yoff+win_ysize - self.raster_y_size)
for xi_copy in xrange(win_xsize):
for yi_copy in xrange(win_ysize):
block_array[yi_copy, xi_copy] = double_buffer[
(yi_copy << self.block_xbits) + xi_copy]
raster_band.WriteArray(
block_array[0:win_ysize, 0:win_xsize],
xoff=xoff, yoff=yoff)
PyMem_Free(double_buffer)
removed_value_list.pop_front()
if self.write_mode:
raster_band = None
raster = None
cpdef calculate_local_recharge(
precip_path_list, et0_path_list, qf_m_path_list, flow_dir_mfd_path,
kc_path_list, alpha_month_map, float beta_i, float gamma, stream_path,
target_li_path, target_li_avail_path, target_l_sum_avail_path,
target_aet_path, target_pi_path):
target_aet_path, target_pi_path, algorithm):
"""
Calculate the rasters defined by equations [3]-[7].
@ -414,291 +63,67 @@ cpdef calculate_local_recharge(
None.
"""
cdef int i_n, flow_dir_nodata, flow_dir_mfd
cdef int peak_pixel
cdef long xs, ys, xs_root, ys_root, xoff, yoff
cdef int flow_dir_s
cdef long xi, yi, xj, yj
cdef int flow_dir_j, p_ij_base
cdef long win_xsize, win_ysize
cdef int n_dir
cdef long raster_x_size, raster_y_size
cdef double pet_m, p_m, qf_m, et0_m, aet_i, p_i, qf_i, l_i, l_avail_i
cdef float qf_nodata, kc_nodata
cdef int j_neighbor_end_index, mfd_dir_sum
cdef float mfd_direction_array[8]
cdef queue[pair[long, long]] work_queue
cdef _ManagedRaster et0_m_raster, qf_m_raster, kc_m_raster
cdef numpy.ndarray[numpy.npy_float32, ndim=1] alpha_month_array = (
numpy.array(
[x[1] for x in sorted(alpha_month_map.items())],
dtype=numpy.float32))
# used for time-delayed logging
cdef time_t last_log_time
last_log_time = ctime(NULL)
# we know the PyGeoprocessing MFD raster flow dir type is a 32 bit int.
flow_dir_raster_info = pygeoprocessing.get_raster_info(flow_dir_mfd_path)
flow_dir_nodata = flow_dir_raster_info['nodata'][0]
raster_x_size, raster_y_size = flow_dir_raster_info['raster_size']
cdef _ManagedRaster flow_raster = _ManagedRaster(flow_dir_mfd_path, 1, 0)
# make sure that user input nodata values are defined
# set to -1 if not defined
# precipitation and evapotranspiration data should
# always be non-negative
et0_m_raster_list = []
et0_m_nodata_list = []
for et0_path in et0_path_list:
et0_m_raster_list.append(_ManagedRaster(et0_path, 1, 0))
nodata = pygeoprocessing.get_raster_info(et0_path)['nodata'][0]
if nodata is None:
nodata = -1
et0_m_nodata_list.append(nodata)
precip_m_raster_list = []
precip_m_nodata_list = []
for precip_m_path in precip_path_list:
precip_m_raster_list.append(_ManagedRaster(precip_m_path, 1, 0))
nodata = pygeoprocessing.get_raster_info(precip_m_path)['nodata'][0]
if nodata is None:
nodata = -1
precip_m_nodata_list.append(nodata)
qf_m_raster_list = []
qf_m_nodata_list = []
for qf_m_path in qf_m_path_list:
qf_m_raster_list.append(_ManagedRaster(qf_m_path, 1, 0))
qf_m_nodata_list.append(
pygeoprocessing.get_raster_info(qf_m_path)['nodata'][0])
kc_m_raster_list = []
kc_m_nodata_list = []
for kc_m_path in kc_path_list:
kc_m_raster_list.append(_ManagedRaster(kc_m_path, 1, 0))
kc_m_nodata_list.append(
pygeoprocessing.get_raster_info(kc_m_path)['nodata'][0])
cdef vector[float] alpha_values
cdef vector[char*] et0_paths
cdef vector[char*] precip_paths
cdef vector[char*] qf_paths
cdef vector[char*] kc_paths
encoded_et0_paths = [p.encode('utf-8') for p in et0_path_list]
encoded_precip_paths = [p.encode('utf-8') for p in precip_path_list]
encoded_qf_paths = [p.encode('utf-8') for p in qf_m_path_list]
encoded_kc_paths = [p.encode('utf-8') for p in kc_path_list]
for i in range(12):
et0_paths.push_back(encoded_et0_paths[i])
precip_paths.push_back(encoded_precip_paths[i])
qf_paths.push_back(encoded_qf_paths[i])
kc_paths.push_back(encoded_kc_paths[i])
alpha_values.push_back(alpha_month_map[i + 1])
target_nodata = -1e32
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_li_path, gdal.GDT_Float32, [target_nodata],
fill_value_list=[target_nodata])
cdef _ManagedRaster target_li_raster = _ManagedRaster(
target_li_path, 1, 1)
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_li_avail_path, gdal.GDT_Float32,
[target_nodata], fill_value_list=[target_nodata])
cdef _ManagedRaster target_li_avail_raster = _ManagedRaster(
target_li_avail_path, 1, 1)
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_l_sum_avail_path, gdal.GDT_Float32,
[target_nodata], fill_value_list=[target_nodata])
cdef _ManagedRaster target_l_sum_avail_raster = _ManagedRaster(
target_l_sum_avail_path, 1, 1)
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_aet_path, gdal.GDT_Float32, [target_nodata],
fill_value_list=[target_nodata])
cdef _ManagedRaster target_aet_raster = _ManagedRaster(
target_aet_path, 1, 1)
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_pi_path, gdal.GDT_Float32, [target_nodata],
fill_value_list=[target_nodata])
cdef _ManagedRaster target_pi_raster = _ManagedRaster(
target_pi_path, 1, 1)
args = [
precip_paths,
et0_paths,
qf_paths,
flow_dir_mfd_path.encode('utf-8'),
kc_paths,
alpha_values,
beta_i,
gamma,
stream_path.encode('utf-8'),
target_li_path.encode('utf-8'),
target_li_avail_path.encode('utf-8'),
target_l_sum_avail_path.encode('utf-8'),
target_aet_path.encode('utf-8'),
target_pi_path.encode('utf-8')]
for offset_dict in pygeoprocessing.iterblocks(
(flow_dir_mfd_path, 1), offset_only=True, largest_block=0):
win_xsize = offset_dict['win_xsize']
win_ysize = offset_dict['win_ysize']
xoff = offset_dict['xoff']
yoff = offset_dict['yoff']
if ctime(NULL) - last_log_time > 5.0:
last_log_time = ctime(NULL)
current_pixel = xoff + yoff * raster_x_size
LOGGER.info(
'peak point detection %.2f%% complete',
100.0 * current_pixel / <float>(
raster_x_size * raster_y_size))
# search block for a peak pixel where no other pixel drains to it.
for ys in xrange(win_ysize):
ys_root = yoff+ys
for xs in xrange(win_xsize):
xs_root = xoff+xs
flow_dir_s = <int>flow_raster.get(xs_root, ys_root)
if flow_dir_s == flow_dir_nodata:
continue
# search neighbors for downhill or nodata
peak_pixel = 1
for n_dir in xrange(8):
# searching around the pattern:
# 321
# 4x0
# 567
xj = xs_root+NEIGHBOR_OFFSET_ARRAY[2*n_dir]
yj = ys_root+NEIGHBOR_OFFSET_ARRAY[2*n_dir+1]
if (xj < 0 or xj >= raster_x_size or
yj < 0 or yj >= raster_y_size):
continue
flow_dir_j = <int>flow_raster.get(xj, yj)
if (0xF & (flow_dir_j >> (
4 * FLOW_DIR_REVERSE_DIRECTION[n_dir]))):
# pixel flows inward, not a peak
peak_pixel = 0
break
if peak_pixel:
work_queue.push(
pair[long, long](xs_root, ys_root))
while work_queue.size() > 0:
xi = work_queue.front().first
yi = work_queue.front().second
work_queue.pop()
l_sum_avail_i = target_l_sum_avail_raster.get(xi, yi)
if not is_close(l_sum_avail_i, target_nodata):
# already defined
continue
# Equation 7, calculate L_sum_avail_i if possible, skip
# otherwise
upslope_defined = 1
# initialize to 0 so we indicate we haven't tracked any
# mfd values yet
j_neighbor_end_index = 0
mfd_dir_sum = 0
for n_dir in xrange(8):
if not upslope_defined:
break
# searching around the pattern:
# 321
# 4x0
# 567
xj = xi+NEIGHBOR_OFFSET_ARRAY[2*n_dir]
yj = yi+NEIGHBOR_OFFSET_ARRAY[2*n_dir+1]
if (xj < 0 or xj >= raster_x_size or
yj < 0 or yj >= raster_y_size):
continue
p_ij_base = (<int>flow_raster.get(xj, yj) >> (
4 * FLOW_DIR_REVERSE_DIRECTION[n_dir])) & 0xF
if p_ij_base:
mfd_dir_sum += p_ij_base
# pixel flows inward, check upslope
l_sum_avail_j = target_l_sum_avail_raster.get(
xj, yj)
if is_close(l_sum_avail_j, target_nodata):
upslope_defined = 0
break
l_avail_j = target_li_avail_raster.get(
xj, yj)
# A step of Equation 7
mfd_direction_array[j_neighbor_end_index] = (
l_sum_avail_j + l_avail_j) * p_ij_base
j_neighbor_end_index += 1
# calculate l_sum_avail_i by summing all the valid
# directions then normalizing by the sum of the mfd
# direction weights (Equation 8)
if upslope_defined:
l_sum_avail_i = 0.0
# Equation 7
if j_neighbor_end_index > 0:
# we can have no upslope, and then why would we
# divide?
for index in range(j_neighbor_end_index):
l_sum_avail_i += mfd_direction_array[index]
l_sum_avail_i /= <float>mfd_dir_sum
target_l_sum_avail_raster.set(xi, yi, l_sum_avail_i)
else:
# if not defined, we'll get it on another pass
continue
aet_i = 0
p_i = 0
qf_i = 0
for m_index in range(12):
precip_m_raster = (
<_ManagedRaster?>precip_m_raster_list[m_index])
qf_m_raster = (
<_ManagedRaster?>qf_m_raster_list[m_index])
et0_m_raster = (
<_ManagedRaster?>et0_m_raster_list[m_index])
kc_m_raster = (
<_ManagedRaster?>kc_m_raster_list[m_index])
et0_nodata = et0_m_nodata_list[m_index]
precip_nodata = precip_m_nodata_list[m_index]
qf_nodata = qf_m_nodata_list[m_index]
kc_nodata = kc_m_nodata_list[m_index]
p_m = precip_m_raster.get(xi, yi)
if not is_close(p_m, precip_nodata):
p_i += p_m
else:
p_m = 0
qf_m = qf_m_raster.get(xi, yi)
if not is_close(qf_m, qf_nodata):
qf_i += qf_m
else:
qf_m = 0
kc_m = kc_m_raster.get(xi, yi)
pet_m = 0
et0_m = et0_m_raster.get(xi, yi)
if not (
is_close(kc_m, kc_nodata) or
is_close(et0_m, et0_nodata)):
# Equation 6
pet_m = kc_m * et0_m
# Equation 4/5
aet_i += min(
pet_m,
p_m - qf_m +
alpha_month_array[m_index]*beta_i*l_sum_avail_i)
target_pi_raster.set(xi, yi, p_i)
target_aet_raster.set(xi, yi, aet_i)
l_i = (p_i - qf_i - aet_i)
# Equation 8
l_avail_i = min(gamma*l_i, l_i)
target_li_raster.set(xi, yi, l_i)
target_li_avail_raster.set(xi, yi, l_avail_i)
flow_dir_mfd = <int>flow_raster.get(xi, yi)
for i_n in range(8):
if ((flow_dir_mfd >> (i_n * 4)) & 0xF) == 0:
# no flow in that direction
continue
xi_n = xi+NEIGHBOR_OFFSET_ARRAY[2*i_n]
yi_n = yi+NEIGHBOR_OFFSET_ARRAY[2*i_n+1]
if (xi_n < 0 or xi_n >= raster_x_size or
yi_n < 0 or yi_n >= raster_y_size):
continue
work_queue.push(pair[long, long](xi_n, yi_n))
if algorithm == 'MFD':
run_calculate_local_recharge[MFD](*args)
else: # D8
run_calculate_local_recharge[D8](*args)
def route_baseflow_sum(
flow_dir_mfd_path, l_path, l_avail_path, l_sum_path,
stream_path, target_b_path, target_b_sum_path):
flow_dir_path, l_path, l_avail_path, l_sum_path,
stream_path, target_b_path, target_b_sum_path, algorithm):
"""Route Baseflow through MFD as described in Equation 11.
Args:
flow_dir_mfd_path (string): path to a pygeoprocessing multiple flow
flow_dir_path (string): path to a pygeoprocessing multiple flow
direction raster.
l_path (string): path to local recharge raster.
l_avail_path (string): path to local recharge raster that shows
@ -713,167 +138,30 @@ def route_baseflow_sum(
Returns:
None.
"""
# used for time-delayed logging
cdef time_t last_log_time
last_log_time = ctime(NULL)
cdef float target_nodata = -1e32
cdef int stream_val, outlet
cdef float b_i, b_sum_i, l_j, l_avail_j, l_sum_j
cdef long xi, yi, xj, yj
cdef int flow_dir_i, p_ij_base
cdef int mfd_dir_sum, flow_dir_nodata
cdef long raster_x_size, raster_y_size, xs_root, ys_root, xoff, yoff
cdef int n_dir
cdef int xs, ys, flow_dir_s, win_xsize, win_ysize
cdef int stream_nodata
cdef stack[pair[long, long]] work_stack
# we know the PyGeoprocessing MFD raster flow dir type is a 32 bit int.
flow_dir_raster_info = pygeoprocessing.get_raster_info(flow_dir_mfd_path)
flow_dir_nodata = flow_dir_raster_info['nodata'][0]
raster_x_size, raster_y_size = flow_dir_raster_info['raster_size']
stream_nodata = pygeoprocessing.get_raster_info(stream_path)['nodata'][0]
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_b_sum_path, gdal.GDT_Float32,
flow_dir_path, target_b_sum_path, gdal.GDT_Float32,
[target_nodata], fill_value_list=[target_nodata])
pygeoprocessing.new_raster_from_base(
flow_dir_mfd_path, target_b_path, gdal.GDT_Float32,
flow_dir_path, target_b_path, gdal.GDT_Float32,
[target_nodata], fill_value_list=[target_nodata])
cdef _ManagedRaster target_b_sum_raster = _ManagedRaster(
target_b_sum_path, 1, 1)
cdef _ManagedRaster target_b_raster = _ManagedRaster(
target_b_path, 1, 1)
cdef _ManagedRaster l_raster = _ManagedRaster(l_path, 1, 0)
cdef _ManagedRaster l_avail_raster = _ManagedRaster(l_avail_path, 1, 0)
cdef _ManagedRaster l_sum_raster = _ManagedRaster(l_sum_path, 1, 0)
cdef _ManagedRaster flow_dir_mfd_raster = _ManagedRaster(
flow_dir_mfd_path, 1, 0)
cdef _ManagedRaster stream_raster = _ManagedRaster(stream_path, 1, 0)
current_pixel = 0
for offset_dict in pygeoprocessing.iterblocks(
(flow_dir_mfd_path, 1), offset_only=True, largest_block=0):
win_xsize = offset_dict['win_xsize']
win_ysize = offset_dict['win_ysize']
xoff = offset_dict['xoff']
yoff = offset_dict['yoff']
# search block for a peak pixel where no other pixel drains to it.
for ys in xrange(win_ysize):
ys_root = yoff+ys
for xs in xrange(win_xsize):
xs_root = xoff+xs
flow_dir_s = <int>flow_dir_mfd_raster.get(xs_root, ys_root)
if flow_dir_s == flow_dir_nodata:
current_pixel += 1
continue
outlet = 1
for n_dir in xrange(8):
if (flow_dir_s >> (n_dir * 4)) & 0xF:
# flows in this direction
xj = xs_root+NEIGHBOR_OFFSET_ARRAY[2*n_dir]
yj = ys_root+NEIGHBOR_OFFSET_ARRAY[2*n_dir+1]
if (xj < 0 or xj >= raster_x_size or
yj < 0 or yj >= raster_y_size):
continue
stream_val = <int>stream_raster.get(xj, yj)
if stream_val != stream_nodata:
outlet = 0
break
if not outlet:
continue
work_stack.push(pair[long, long](xs_root, ys_root))
while work_stack.size() > 0:
xi = work_stack.top().first
yi = work_stack.top().second
work_stack.pop()
b_sum_i = target_b_sum_raster.get(xi, yi)
if not is_close(b_sum_i, target_nodata):
continue
if ctime(NULL) - last_log_time > 5.0:
last_log_time = ctime(NULL)
LOGGER.info(
'route base flow %.2f%% complete',
100.0 * current_pixel / <float>(
raster_x_size * raster_y_size))
b_sum_i = 0.0
mfd_dir_sum = 0
downslope_defined = 1
flow_dir_i = <int>flow_dir_mfd_raster.get(xi, yi)
if flow_dir_i == flow_dir_nodata:
LOGGER.error("flow dir nodata? this makes no sense")
continue
for n_dir in xrange(8):
if not downslope_defined:
break
# searching around the pattern:
# 321
# 4x0
# 567
p_ij_base = (flow_dir_i >> (4*n_dir)) & 0xF
if p_ij_base:
mfd_dir_sum += p_ij_base
xj = xi+NEIGHBOR_OFFSET_ARRAY[2*n_dir]
yj = yi+NEIGHBOR_OFFSET_ARRAY[2*n_dir+1]
if (xj < 0 or xj >= raster_x_size or
yj < 0 or yj >= raster_y_size):
continue
stream_val = <int>stream_raster.get(xj, yj)
if stream_val:
b_sum_i += p_ij_base
else:
b_sum_j = target_b_sum_raster.get(xj, yj)
if is_close(b_sum_j, target_nodata):
downslope_defined = 0
break
l_j = l_raster.get(xj, yj)
l_avail_j = l_avail_raster.get(xj, yj)
l_sum_j = l_sum_raster.get(xj, yj)
if l_sum_j != 0 and (l_sum_j - l_j) != 0:
b_sum_i += p_ij_base * (
(1-l_avail_j / l_sum_j)*(
b_sum_j / (l_sum_j - l_j)))
else:
b_sum_i += p_ij_base
if not downslope_defined:
continue
l_sum_i = l_sum_raster.get(xi, yi)
if mfd_dir_sum > 0:
# normalize by mfd weight
b_sum_i = l_sum_i * b_sum_i / <float>mfd_dir_sum
target_b_sum_raster.set(xi, yi, b_sum_i)
l_i = l_raster.get(xi, yi)
if l_sum_i != 0:
b_i = max(b_sum_i * l_i / l_sum_i, 0.0)
else:
b_i = 0.0
target_b_raster.set(xi, yi, b_i)
current_pixel += 1
for n_dir in xrange(8):
# searching upslope for pixels that flow in
# 321
# 4x0
# 567
xj = xi+NEIGHBOR_OFFSET_ARRAY[2*n_dir]
yj = yi+NEIGHBOR_OFFSET_ARRAY[2*n_dir+1]
if (xj < 0 or xj >= raster_x_size or
yj < 0 or yj >= raster_y_size):
continue
flow_dir_j = <int>flow_dir_mfd_raster.get(xj, yj)
if (0xF & (flow_dir_j >> (
4 * FLOW_DIR_REVERSE_DIRECTION[n_dir]))):
# pixel flows here, push on queue
work_stack.push(pair[long, long](xj, yj))
if algorithm == 'MFD':
run_route_baseflow_sum[MFD](
flow_dir_path.encode('utf-8'),
l_path.encode('utf-8'),
l_avail_path.encode('utf-8'),
l_sum_path.encode('utf-8'),
stream_path.encode('utf-8'),
target_b_path.encode('utf-8'),
target_b_sum_path.encode('utf-8'))
else: # D8
run_route_baseflow_sum[D8](
flow_dir_path.encode('utf-8'),
l_path.encode('utf-8'),
l_avail_path.encode('utf-8'),
l_sum_path.encode('utf-8'),
stream_path.encode('utf-8'),
target_b_path.encode('utf-8'),
target_b_sum_path.encode('utf-8'))

466
src/natcap/invest/seasonal_water_yield/swy.h Normal file
View File

@ -0,0 +1,466 @@
#include <algorithm>
#include <stack>
#include <queue>
#include <ctime>
#include "ManagedRaster.h"
// Calculate the rasters defined by equations [3]-[7].
//
// Note all input rasters must be in the same coordinate system and
// have the same dimensions.
//
// Args:
// precip_paths: paths to monthly precipitation rasters. (model input)
// et0_paths: paths to monthly ET0 rasters. (model input)
// qf_m_paths: paths to monthly quickflow rasters calculated by
// Equation [1].
// flow_dir_path: path to a flow direction raster (MFD or D8). Indicate MFD
// or D8 with the template argument.
// kc_paths: list of rasters of the monthly crop factor for the pixel.
// alpha_values: list of monthly alpha values (fraction of upslope annual
// available recharge that is available in each month)
// beta_i: fraction of the upgradient subsidy that is available
// for downgradient evapotranspiration.
// gamma: the fraction of pixel recharge that is available to
// downgradient pixels.
// stream_path: path to the stream raster where 1 is a stream,
// 0 is not, and nodata is outside of the DEM.
// target_li_path: created by this call, path to local recharge
// derived from the annual water budget. (Equation 3).
// target_li_avail_path: created by this call, path to raster
// indicating available recharge to a pixel.
// target_l_sum_avail_path: created by this call, the recursive
// upslope accumulation of target_li_avail_path.
// target_aet_path: created by this call, the annual actual
// evapotranspiration.
// target_pi_path: created by this call, the annual precipitation on
// a pixel.
template<class T>
void run_calculate_local_recharge(
vector<char*> precip_paths,
vector<char*> et0_paths,
vector<char*> qf_m_paths,
char* flow_dir_path,
vector<char*> kc_paths,
vector<float> alpha_values,
float beta_i,
float gamma,
char* stream_path,
char* target_li_path,
char* target_li_avail_path,
char* target_l_sum_avail_path,
char* target_aet_path,
char* target_pi_path) {
long xs_root, ys_root, xoff, yoff;
long xi, yi, mfd_dir_sum;
long win_xsize, win_ysize;
double kc_m, pet_m, p_m, qf_m, et0_m, aet_i, p_i, qf_i, l_i;
double l_avail_i, l_avail_j, l_sum_avail_i, l_sum_avail_j;
bool upslope_defined;
queue<pair<long, long>> work_queue;
UpslopeNeighborsNoDivide<T> up_neighbors;
DownslopeNeighbors<T> dn_neighbors;
ManagedFlowDirRaster<T> flow_dir_raster = ManagedFlowDirRaster<T>(
flow_dir_path, 1, 0);
NeighborTuple neighbor;
time_t last_log_time = time(NULL);
unsigned long n_pixels_processed = 0;
float total_n_pixels = flow_dir_raster.raster_x_size * flow_dir_raster.raster_y_size;
// make sure that user input nodata values are defined
// set to -1 if not defined
// precipitation and evapotranspiration data should
// always be non-negative
vector<ManagedRaster> et0_m_rasters;
vector<double> et0_m_nodata_list;
for (auto et0_m_path: et0_paths) {
ManagedRaster et0_raster = ManagedRaster(et0_m_path, 1, 0);
et0_m_rasters.push_back(et0_raster);
if (et0_raster.hasNodata) {
et0_m_nodata_list.push_back(et0_raster.nodata);
} else {
et0_m_nodata_list.push_back(-1);
}
}
vector<ManagedRaster> precip_m_rasters;
vector<double> precip_m_nodata_list;
for (auto precip_m_path: precip_paths) {
ManagedRaster precip_raster = ManagedRaster(precip_m_path, 1, 0);
precip_m_rasters.push_back(precip_raster);
if (precip_raster.hasNodata) {
precip_m_nodata_list.push_back(precip_raster.nodata);
} else {
precip_m_nodata_list.push_back(-1);
}
}
vector<ManagedRaster> qf_m_rasters;
for (auto qf_m_path: qf_m_paths) {
qf_m_rasters.push_back(ManagedRaster(qf_m_path, 1, 0));
}
vector<ManagedRaster> kc_m_rasters;
for (auto kc_m_path: kc_paths) {
kc_m_rasters.push_back(ManagedRaster(kc_m_path, 1, 0));
}
ManagedRaster target_li_raster = ManagedRaster(target_li_path, 1, 1);
ManagedRaster target_li_avail_raster = ManagedRaster(target_li_avail_path, 1, 1);
ManagedRaster target_l_sum_avail_raster = ManagedRaster(target_l_sum_avail_path, 1, 1);
ManagedRaster target_aet_raster = ManagedRaster(target_aet_path, 1, 1);
ManagedRaster target_pi_raster = ManagedRaster(target_pi_path, 1, 1);
double target_nodata = -1e32;
// efficient way to calculate ceiling division:
// a divided by b rounded up = (a + (b - 1)) / b
// note that / represents integer floor division
// https://stackoverflow.com/a/62032709/14451410
int n_col_blocks = (flow_dir_raster.raster_x_size + (flow_dir_raster.block_xsize - 1)) / flow_dir_raster.block_xsize;
int n_row_blocks = (flow_dir_raster.raster_y_size + (flow_dir_raster.block_ysize - 1)) / flow_dir_raster.block_ysize;
for (int row_block_index = 0; row_block_index < n_row_blocks; row_block_index++) {
yoff = row_block_index * flow_dir_raster.block_ysize;
win_ysize = flow_dir_raster.raster_y_size - yoff;
if (win_ysize > flow_dir_raster.block_ysize) {
win_ysize = flow_dir_raster.block_ysize;
}
for (int col_block_index = 0; col_block_index < n_col_blocks; col_block_index++) {
xoff = col_block_index * flow_dir_raster.block_xsize;
win_xsize = flow_dir_raster.raster_x_size - xoff;
if (win_xsize > flow_dir_raster.block_xsize) {
win_xsize = flow_dir_raster.block_xsize;
}
if (time(NULL) - last_log_time > 5) {
last_log_time = time(NULL);
log_msg(
LogLevel::info,
"Local recharge " + std::to_string(
100 * n_pixels_processed / total_n_pixels
) + " complete"
);
}
for (int row_index = 0; row_index < win_ysize; row_index++) {
ys_root = yoff + row_index;
for (int col_index = 0; col_index < win_xsize; col_index++) {
xs_root = xoff + col_index;
if (flow_dir_raster.get(xs_root, ys_root) == flow_dir_raster.nodata) {
continue;
}
if (flow_dir_raster.is_local_high_point(xs_root, ys_root)) {
work_queue.push(pair<long, long>(xs_root, ys_root));
}
while (work_queue.size() > 0) {
xi = work_queue.front().first;
yi = work_queue.front().second;
work_queue.pop();
l_sum_avail_i = target_l_sum_avail_raster.get(xi, yi);
if (not is_close(l_sum_avail_i, target_nodata)) {
// already defined
continue;
}
// Equation 7, calculate L_sum_avail_i if possible, skip
// otherwise
upslope_defined = true;
// initialize to 0 so we indicate we haven't tracked any
// mfd values yet
l_sum_avail_i = 0.0;
mfd_dir_sum = 0;
up_neighbors = UpslopeNeighborsNoDivide<T>(Pixel<T>(flow_dir_raster, xi, yi));
for (auto neighbor: up_neighbors) {
// pixel flows inward, check upslope
l_sum_avail_j = target_l_sum_avail_raster.get(
neighbor.x, neighbor.y);
if (is_close(l_sum_avail_j, target_nodata)) {
upslope_defined = false;
break;
}
l_avail_j = target_li_avail_raster.get(
neighbor.x, neighbor.y);
// A step of Equation 7
l_sum_avail_i += (
l_sum_avail_j + l_avail_j) * neighbor.flow_proportion;
mfd_dir_sum += static_cast<int>(neighbor.flow_proportion);
}
// calculate l_sum_avail_i by summing all the valid
// directions then normalizing by the sum of the mfd
// direction weights (Equation 8)
if (upslope_defined) {
// Equation 7
if (mfd_dir_sum > 0) {
l_sum_avail_i /= static_cast<float>(mfd_dir_sum);
}
target_l_sum_avail_raster.set(xi, yi, l_sum_avail_i);
} else {
// if not defined, we'll get it on another pass
continue;
}
aet_i = 0;
p_i = 0;
qf_i = 0;
for (int m_index = 0; m_index < 12; m_index++) {
p_m = precip_m_rasters[m_index].get(xi, yi);
if (not is_close(p_m, precip_m_rasters[m_index].nodata)) {
p_i += p_m;
} else {
p_m = 0;
}
qf_m = qf_m_rasters[m_index].get(xi, yi);
if (not is_close(qf_m, qf_m_rasters[m_index].nodata)) {
qf_i += qf_m;
} else {
qf_m = 0;
}
kc_m = kc_m_rasters[m_index].get(xi, yi);
pet_m = 0;
et0_m = et0_m_rasters[m_index].get(xi, yi);
if (not (
is_close(kc_m, kc_m_rasters[m_index].nodata) or
is_close(et0_m, et0_m_rasters[m_index].nodata))) {
// Equation 6
pet_m = kc_m * et0_m;
}
// Equation 4/5
aet_i += min(
pet_m,
p_m - qf_m +
alpha_values[m_index]*beta_i*l_sum_avail_i);
}
l_i = (p_i - qf_i - aet_i);
l_avail_i = min(gamma * l_i, l_i);
target_pi_raster.set(xi, yi, p_i);
target_aet_raster.set(xi, yi, aet_i);
target_li_raster.set(xi, yi, l_i);
target_li_avail_raster.set(xi, yi, l_avail_i);
dn_neighbors = DownslopeNeighbors<T>(Pixel<T>(flow_dir_raster, xi, yi));
for (auto neighbor: dn_neighbors) {
work_queue.push(pair<long, long>(neighbor.x, neighbor.y));
}
}
}
}
n_pixels_processed += win_xsize * win_ysize;
}
}
flow_dir_raster.close();
target_li_raster.close();
target_li_avail_raster.close();
target_l_sum_avail_raster.close();
target_aet_raster.close();
target_pi_raster.close();
for (int i = 0; i < 12; i++) {
et0_m_rasters[i].close();
precip_m_rasters[i].close();
qf_m_rasters[i].close();
kc_m_rasters[i].close();
}
log_msg(LogLevel::info, "Local recharge 100% complete");
}
// Route Baseflow as described in Equation 11.
// Args:
// flow_dir_path: path to a MFD or D8 flow direction raster.
// l_path: path to local recharge raster.
// l_avail_path: path to local recharge raster that shows
// recharge available to the pixel.
// l_sum_path: path to upslope sum of l_path.
// stream_path: path to stream raster, 1 stream, 0 no stream,
// and nodata.
// target_b_path: path to created raster for per-pixel baseflow.
// target_b_sum_path: path to created raster for per-pixel
// upslope sum of baseflow.
template<class T>
void run_route_baseflow_sum(
char* flow_dir_path,
char* l_path,
char* l_avail_path,
char* l_sum_path,
char* stream_path,
char* target_b_path,
char* target_b_sum_path) {
float target_nodata = static_cast<float>(-1e32);
double b_i, b_sum_i, b_sum_j, l_j, l_avail_j, l_sum_j;
double l_i, l_sum_i;
long xi, yi, flow_dir_sum;
long xs_root, ys_root, xoff, yoff;
int win_xsize, win_ysize;
stack<pair<long, long>> work_stack;
bool outlet, downslope_defined;
ManagedRaster target_b_sum_raster = ManagedRaster(target_b_sum_path, 1, 1);
ManagedRaster target_b_raster = ManagedRaster(target_b_path, 1, 1);
ManagedRaster l_raster = ManagedRaster(l_path, 1, 0);
ManagedRaster l_avail_raster = ManagedRaster(l_avail_path, 1, 0);
ManagedRaster l_sum_raster = ManagedRaster(l_sum_path, 1, 0);
ManagedFlowDirRaster<T> flow_dir_raster = ManagedFlowDirRaster<T>(flow_dir_path, 1, 0);
ManagedRaster stream_raster = ManagedRaster(stream_path, 1, 0);
UpslopeNeighbors<T> up_neighbors;
DownslopeNeighbors<T> dn_neighbors;
DownslopeNeighborsNoSkip<T> dn_neighbors_no_skip;
NeighborTuple neighbor;
time_t last_log_time = time(NULL);
unsigned long current_pixel = 0;
float total_n_pixels = flow_dir_raster.raster_x_size * flow_dir_raster.raster_y_size;
// efficient way to calculate ceiling division:
// a divided by b rounded up = (a + (b - 1)) / b
// note that / represents integer floor division
// https://stackoverflow.com/a/62032709/14451410
int n_col_blocks = (flow_dir_raster.raster_x_size + (flow_dir_raster.block_xsize - 1)) / flow_dir_raster.block_xsize;
int n_row_blocks = (flow_dir_raster.raster_y_size + (flow_dir_raster.block_ysize - 1)) / flow_dir_raster.block_ysize;
for (int row_block_index = 0; row_block_index < n_row_blocks; row_block_index++) {
yoff = row_block_index * flow_dir_raster.block_ysize;
win_ysize = flow_dir_raster.raster_y_size - yoff;
if (win_ysize > flow_dir_raster.block_ysize) {
win_ysize = flow_dir_raster.block_ysize;
}
for (int col_block_index = 0; col_block_index < n_col_blocks; col_block_index++) {
xoff = col_block_index * flow_dir_raster.block_xsize;
win_xsize = flow_dir_raster.raster_x_size - xoff;
if (win_xsize > flow_dir_raster.block_xsize) {
win_xsize = flow_dir_raster.block_xsize;
}
for (int row_index = 0; row_index < win_ysize; row_index++) {
ys_root = yoff + row_index;
for (int col_index = 0; col_index < win_xsize; col_index++) {
xs_root = xoff + col_index;
if (static_cast<int>(flow_dir_raster.get(xs_root, ys_root)) ==
static_cast<int>(flow_dir_raster.nodata)) {
current_pixel += 1;
continue;
}
// search for a pixel that has no downslope neighbors,
// or whose downslope neighbors all have nodata in the stream raster (?)
outlet = true;
dn_neighbors = DownslopeNeighbors<T>(Pixel<T>(flow_dir_raster, xs_root, ys_root));
for (auto neighbor: dn_neighbors) {
if (static_cast<int>(stream_raster.get(neighbor.x, neighbor.y)) !=
static_cast<int>(stream_raster.nodata)) {
outlet = 0;
break;
}
}
if (not outlet) {
continue;
}
work_stack.push(pair<long, long>(xs_root, ys_root));
while (work_stack.size() > 0) {
xi = work_stack.top().first;
yi = work_stack.top().second;
work_stack.pop();
b_sum_i = target_b_sum_raster.get(xi, yi);
if (not is_close(b_sum_i, target_nodata)) {
continue;
}
if (time(NULL) - last_log_time > 5) {
last_log_time = time(NULL);
log_msg(
LogLevel::info,
"Baseflow " + std::to_string(
100 * current_pixel / total_n_pixels
) + " complete"
);
}
b_sum_i = 0;
downslope_defined = true;
dn_neighbors_no_skip = DownslopeNeighborsNoSkip<T>(Pixel<T>(flow_dir_raster, xi, yi));
flow_dir_sum = 0;
for (auto neighbor: dn_neighbors_no_skip) {
flow_dir_sum += static_cast<long>(neighbor.flow_proportion);
if (neighbor.x < 0 or neighbor.x >= flow_dir_raster.raster_x_size or
neighbor.y < 0 or neighbor.y >= flow_dir_raster.raster_y_size) {
continue;
}
if (static_cast<int>(stream_raster.get(neighbor.x, neighbor.y))) {
b_sum_i += neighbor.flow_proportion;
} else {
b_sum_j = target_b_sum_raster.get(neighbor.x, neighbor.y);
if (is_close(b_sum_j, target_nodata)) {
downslope_defined = false;
break;
}
l_j = l_raster.get(neighbor.x, neighbor.y);
l_avail_j = l_avail_raster.get(neighbor.x, neighbor.y);
l_sum_j = l_sum_raster.get(neighbor.x, neighbor.y);
if (l_sum_j != 0 and (l_sum_j - l_j) != 0) {
b_sum_i += neighbor.flow_proportion * (
(1 - l_avail_j / l_sum_j) * (
b_sum_j / (l_sum_j - l_j)));
} else {
b_sum_i += neighbor.flow_proportion;
}
}
}
if (not downslope_defined) {
continue;
}
l_i = l_raster.get(xi, yi);
l_sum_i = l_sum_raster.get(xi, yi);
if (flow_dir_sum > 0) {
b_sum_i = l_sum_i * b_sum_i / flow_dir_sum;
}
if (l_sum_i != 0) {
b_i = max(b_sum_i * l_i / l_sum_i, 0.0);
} else {
b_i = 0;
}
target_b_raster.set(xi, yi, b_i);
target_b_sum_raster.set(xi, yi, b_sum_i);
current_pixel += 1;
up_neighbors = UpslopeNeighbors<T>(Pixel<T>(flow_dir_raster, xi, yi));
for (auto neighbor: up_neighbors) {
work_stack.push(pair<long, long>(neighbor.x, neighbor.y));
}
}
}
}
}
}
target_b_sum_raster.close();
target_b_raster.close();
l_raster.close();
l_avail_raster.close();
l_sum_raster.close();
flow_dir_raster.close();
stream_raster.close();
log_msg(LogLevel::info, "Baseflow 100% complete");
}

28
src/natcap/invest/seasonal_water_yield/swy.pxd Normal file
View File

@ -0,0 +1,28 @@
from libcpp.vector cimport vector
cdef extern from "swy.h":
void run_calculate_local_recharge[T](
vector[char*], # precip_path_list
vector[char*], # et0_path_list
vector[char*], # qf_m_path_list
char*, # flow_dir_mfd_path
vector[char*], # kc_path_list
vector[float], # alpha_values
float, # beta_i
float, # gamma
char*, # stream_path
char*, # target_li_path
char*, # target_li_avail_path
char*, # target_l_sum_avail_path
char*, # target_aet_path
char* # target_pi_path
) except +
void run_route_baseflow_sum[T](
char*,
char*,
char*,
char*,
char*,
char*,
char*) except +

116
src/natcap/invest/spec_utils.py
View File

@ -1,12 +1,20 @@
import importlib
import json
import logging
import os
import pprint
import geometamaker
import natcap.invest
import pint
from natcap.invest import utils
from . import gettext
from .unit_registry import u
LOGGER = logging.getLogger(__name__)
# Specs for common arg types ##################################################
WORKSPACE = {
"name": gettext("workspace"),
@ -176,6 +184,24 @@ STREAM = {
"bands": {1: {"type": "integer"}}
}
FLOW_DIR_ALGORITHM = {
"flow_dir_algorithm": {
"type": "option_string",
"options": {
"D8": {
"display_name": gettext("D8"),
"description": "D8 flow direction"
},
"MFD": {
"display_name": gettext("MFD"),
"description": "Multiple flow direction"
}
},
"about": gettext("Flow direction algorithm to use."),
"name": gettext("flow direction algorithm")
}
}
# geometry types ##############################################################
# the full list of ogr geometry types is in an enum in
# https://github.com/OSGeo/gdal/blob/master/gdal/ogr/ogr_core.h
@ -585,3 +611,93 @@ def describe_arg_from_name(module_name, *arg_keys):
anchor_name = '-'.join(arg_keys).replace('_', '-')
rst_description = '\n\n'.join(describe_arg_from_spec(arg_name, spec))
return f'.. _{anchor_name}:\n\n{rst_description}'
def write_metadata_file(datasource_path, spec, lineage_statement, keywords_list):
"""Write a metadata sidecar file for an invest output dataset.
Args:
datasource_path (str) - filepath to the invest output
spec (dict) - the invest specification for ``datasource_path``
lineage_statement (str) - string to describe origin of the dataset.
keywords_list (list) - sequence of strings
Returns:
None
"""
resource = geometamaker.describe(datasource_path)
resource.set_lineage(lineage_statement)
# a pre-existing metadata doc could have keywords
words = resource.get_keywords()
resource.set_keywords(set(words + keywords_list))
if 'about' in spec:
resource.set_description(spec['about'])
attr_spec = None
if 'columns' in spec:
attr_spec = spec['columns']
if 'fields' in spec:
attr_spec = spec['fields']
if attr_spec:
for key, value in attr_spec.items():
about = value['about'] if 'about' in value else ''
units = format_unit(value['units']) if 'units' in value else ''
try:
resource.set_field_description(
key, description=about, units=units)
except KeyError as error:
# fields that are in the spec but missing
# from model results because they are conditional.
LOGGER.debug(error)
if 'bands' in spec:
for idx, value in spec['bands'].items():
try:
units = format_unit(spec['bands'][idx]['units'])
except KeyError:
units = ''
resource.set_band_description(idx, units=units)
resource.write()
def generate_metadata(model_module, args_dict):
"""Create metadata for all items in an invest model output workspace.
Args:
model_module (object) - the natcap.invest module containing
the MODEL_SPEC attribute
args_dict (dict) - the arguments dictionary passed to the
model's ``execute`` function.
Returns:
None
"""
file_suffix = utils.make_suffix_string(args_dict, 'results_suffix')
formatted_args = pprint.pformat(args_dict)
lineage_statement = (
f'Created by {model_module.__name__}.execute(\n{formatted_args})\n'
f'Version {natcap.invest.__version__}')
keywords = [model_module.MODEL_SPEC['model_id'], 'InVEST']
def _walk_spec(output_spec, workspace):
for filename, spec_data in output_spec.items():
if 'type' in spec_data and spec_data['type'] == 'directory':
if 'taskgraph.db' in spec_data['contents']:
continue
_walk_spec(
spec_data['contents'],
os.path.join(workspace, filename))
else:
pre, post = os.path.splitext(filename)
full_path = os.path.join(workspace, f'{pre}{file_suffix}{post}')
if os.path.exists(full_path):
try:
write_metadata_file(
full_path, spec_data, lineage_statement, keywords)
except ValueError as error:
# Some unsupported file formats, e.g. html
LOGGER.debug(error)
_walk_spec(model_module.MODEL_SPEC['outputs'], args_dict['workspace_dir'])

1
src/natcap/invest/stormwater.py
View File

@ -27,6 +27,7 @@ UINT16_NODATA = 65535
NONINTEGER_SOILS_RASTER_MESSAGE = 'Soil group raster data type must be integer'
MODEL_SPEC = {
"model_id": "stormwater",
"model_name": MODEL_METADATA["stormwater"].model_title,
"pyname": MODEL_METADATA["stormwater"].pyname,
"userguide": MODEL_METADATA["stormwater"].userguide,

28
src/natcap/invest/ui_server.py
View File

@ -7,6 +7,7 @@ from osgeo import gdal
from flask import Flask
from flask import request
from flask_cors import CORS
import geometamaker
import natcap.invest
from natcap.invest import cli
from natcap.invest import datastack
@ -279,3 +280,30 @@ def log_model_exit():
def get_supported_languages():
"""Return a mapping of supported languages to their display names."""
return json.dumps(natcap.invest.LOCALE_NAME_MAP)
@app.route(f'/{PREFIX}/get_geometamaker_profile', methods=['GET'])
def get_geometamaker_profile():
"""Return the user-profile from geometamaker."""
config = geometamaker.Config()
return config.profile.model_dump()
@app.route(f'/{PREFIX}/set_geometamaker_profile', methods=['POST'])
def set_geometamaker_profile():
"""Set the user-profile for geometamaker.
Body (JSON string): deserializes to a dict with keys:
contact
license
"""
payload = request.get_json()
profile = geometamaker.Profile(**payload)
config = geometamaker.Config()
config.save(profile)
LOGGER.debug(config)
return {
'message': 'Metadata profile saved',
'error': False
}

3
src/natcap/invest/urban_cooling_model.py
View File

@ -30,6 +30,7 @@ TARGET_NODATA = -1
_LOGGING_PERIOD = 5
MODEL_SPEC = {
"model_id": "urban_cooling_model",
"model_name": MODEL_METADATA["urban_cooling_model"].model_title,
"pyname": MODEL_METADATA["urban_cooling_model"].pyname,
"userguide": MODEL_METADATA["urban_cooling_model"].userguide,
@ -531,7 +532,7 @@ def execute(args):
'intersection'),
kwargs={
'base_vector_path_list': [args['aoi_vector_path']],
'raster_align_index': 1,
'raster_align_index': 0,
'target_projection_wkt': lulc_raster_info['projection_wkt']},
target_path_list=aligned_raster_path_list,
task_name='align rasters')

1
src/natcap/invest/urban_flood_risk_mitigation.py
View File

@ -23,6 +23,7 @@ from .unit_registry import u
LOGGER = logging.getLogger(__name__)
MODEL_SPEC = {
"model_id": "urban_flood_risk_mitigation",
"model_name": MODEL_METADATA["urban_flood_risk_mitigation"].model_title,
"pyname": MODEL_METADATA["urban_flood_risk_mitigation"].pyname,
"userguide": MODEL_METADATA["urban_flood_risk_mitigation"].userguide,

1
src/natcap/invest/urban_nature_access.py
View File

@ -40,6 +40,7 @@ RADIUS_OPT_POP_GROUP = 'radius per population group'
POP_FIELD_REGEX = '^pop_'
ID_FIELDNAME = 'adm_unit_id'
MODEL_SPEC = {
'model_id': 'urban_nature_access',
'model_name': MODEL_METADATA['urban_nature_access'].model_title,
'pyname': MODEL_METADATA['urban_nature_access'].pyname,
'userguide': MODEL_METADATA['urban_nature_access'].userguide,

1
src/natcap/invest/utils.py
View File

@ -18,6 +18,7 @@ from osgeo import gdal
from osgeo import osr
from shapely.wkt import loads
LOGGER = logging.getLogger(__name__)
_OSGEO_LOGGER = logging.getLogger('osgeo')
LOG_FMT = (

1
src/natcap/invest/wave_energy.py
View File

@ -132,6 +132,7 @@ CAPTURED_WEM_FIELDS = {
}
MODEL_SPEC = {
"model_id": "wave_energy",
"model_name": MODEL_METADATA["wave_energy"].model_title,
"pyname": MODEL_METADATA["wave_energy"].pyname,
"userguide": MODEL_METADATA["wave_energy"].userguide,

1
src/natcap/invest/wind_energy.py
View File

@ -92,6 +92,7 @@ OUTPUT_WIND_DATA_FIELDS = {
}
MODEL_SPEC = {
"model_id": "wind_energy",
"model_name": MODEL_METADATA["wind_energy"].model_title,
"pyname": MODEL_METADATA["wind_energy"].pyname,
"userguide": MODEL_METADATA["wind_energy"].userguide,

164
tests/test_carbon.py
View File

@ -4,12 +4,14 @@ import unittest
import tempfile
import shutil
import os
import re
from osgeo import gdal
from osgeo import osr
import numpy
import numpy.random
import numpy.testing
import pygeoprocessing
gdal.UseExceptions()
@ -62,7 +64,8 @@ def assert_raster_equal_value(base_raster_path, val_to_compare):
array_to_compare = numpy.empty(base_array.shape)
array_to_compare.fill(val_to_compare)
numpy.testing.assert_allclose(base_array, array_to_compare, rtol=0, atol=1e-6)
numpy.testing.assert_allclose(base_array, array_to_compare,
rtol=0, atol=1e-3)
def make_pools_csv(pools_csv_path):
@ -84,6 +87,28 @@ def make_pools_csv(pools_csv_path):
open_table.write('2,1,5,0,3,"lulc code 3"\n')
def assert_aggregate_result_equal(html_report_path, stat_name, val_to_compare):
"""Assert that the given stat in the HTML report has a specific value.
Args:
html_report_path (str): path to the HTML report generated by the model.
stat_name (str): name of the stat to find. Must match the name listed
in the HTML.
val_to_compare (float): the value to check against.
Returns:
None.
"""
with open(html_report_path) as file:
report = file.read()
pattern = (r'data-summary-stat="'
+ stat_name
+ r'">(\-?\d+\.\d{2})</td>')
match = re.search(pattern, report)
stat_str = match.groups()[0]
assert float(stat_str) == val_to_compare
class CarbonTests(unittest.TestCase):
"""Tests for the Carbon Model."""
@ -106,14 +131,14 @@ class CarbonTests(unittest.TestCase):
'do_valuation': True,
'price_per_metric_ton_of_c': 43.0,
'rate_change': 2.8,
'lulc_cur_year': 2016,
'lulc_fut_year': 2030,
'lulc_bas_year': 2016,
'lulc_alt_year': 2030,
'discount_rate': -7.1,
'n_workers': -1,
}
# Create LULC rasters and pools csv in workspace and add them to args.
lulc_names = ['lulc_cur_path', 'lulc_fut_path', 'lulc_redd_path']
lulc_names = ['lulc_bas_path', 'lulc_alt_path']
for fill_val, lulc_name in enumerate(lulc_names, 1):
args[lulc_name] = os.path.join(args['workspace_dir'],
lulc_name + '.tif')
@ -125,12 +150,35 @@ class CarbonTests(unittest.TestCase):
carbon.execute(args)
# Add assertions for npv for future and REDD scenarios.
# The npv was calculated based on _calculate_npv in carbon.py.
# Ensure every pixel has the correct total C value.
# Baseline: 15 + 10 + 60 + 1 = 86 Mg/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_fut.tif'), -0.3422078)
os.path.join(args['workspace_dir'], 'c_storage_bas.tif'), 86)
# Alternate: 5 + 3 + 20 + 0 = 28 Mg/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_redd.tif'), -0.4602106)
os.path.join(args['workspace_dir'], 'c_storage_alt.tif'), 28)
# Ensure c_changes are correct.
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'c_change_bas_alt.tif'), -58)
# Ensure NPV calculations are correct.
# Valuation constant based on provided args is 59.00136.
# Alternate: 59.00136 * -58 Mg/ha = -3422.079 Mg/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_alt.tif'), -3422.079)
# Ensure aggregate results are correct.
report_path = os.path.join(args['workspace_dir'], 'report.html')
# Raster size is 100 m^2; therefore, raster total is as follows:
# (x Mg / 1 ha) * (1 ha / 10000 m^2) * (100 m^2) = (x / 100) Mg
for (stat, expected_value) in [
('Baseline Carbon Storage', 0.86),
('Alternate Carbon Storage', 0.28),
('Change in Carbon Storage', -0.58),
('Net Present Value of Carbon Change', -34.22),
]:
assert_aggregate_result_equal(report_path, stat, expected_value)
def test_carbon_zero_rates(self):
"""Carbon: test with 0 discount and rate change."""
@ -141,14 +189,14 @@ class CarbonTests(unittest.TestCase):
'do_valuation': True,
'price_per_metric_ton_of_c': 43.0,
'rate_change': 0.0,
'lulc_cur_year': 2016,
'lulc_fut_year': 2030,
'lulc_bas_year': 2016,
'lulc_alt_year': 2030,
'discount_rate': 0.0,
'n_workers': -1,
}
# Create LULC rasters and pools csv in workspace and add them to args.
lulc_names = ['lulc_cur_path', 'lulc_fut_path', 'lulc_redd_path']
lulc_names = ['lulc_bas_path', 'lulc_alt_path']
for fill_val, lulc_name in enumerate(lulc_names, 1):
args[lulc_name] = os.path.join(args['workspace_dir'],
lulc_name + '.tif')
@ -160,31 +208,27 @@ class CarbonTests(unittest.TestCase):
carbon.execute(args)
# Add assertions for npv for future and REDD scenarios.
# carbon change from cur to fut:
# -58 Mg/ha * .0001 ha/pixel * 43 $/Mg = -0.2494 $/pixel
# Add assertions for npv for alternate scenario.
# carbon change from bas to alt:
# -58 Mg/ha * 43 $/Mg = -2494 $/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_fut.tif'), -0.2494)
# carbon change from cur to redd:
# -78 Mg/ha * .0001 ha/pixel * 43 $/Mg = -0.3354 $/pixel
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_redd.tif'), -0.3354)
os.path.join(args['workspace_dir'], 'npv_alt.tif'), -2494)
def test_carbon_future(self):
"""Carbon: regression testing future scenario."""
def test_carbon_alternate_scenario(self):
"""Carbon: regression testing for alternate scenario"""
from natcap.invest import carbon
args = {
'workspace_dir': self.workspace_dir,
'do_valuation': True,
'price_per_metric_ton_of_c': 43.0,
'rate_change': 2.8,
'lulc_cur_year': 2016,
'lulc_fut_year': 2030,
'lulc_bas_year': 2016,
'lulc_alt_year': 2030,
'discount_rate': -7.1,
'n_workers': -1,
}
lulc_names = ['lulc_cur_path', 'lulc_fut_path']
lulc_names = ['lulc_bas_path', 'lulc_alt_path']
for fill_val, lulc_name in enumerate(lulc_names, 1):
args[lulc_name] = os.path.join(args['workspace_dir'],
lulc_name + '.tif')
@ -195,10 +239,12 @@ class CarbonTests(unittest.TestCase):
make_pools_csv(args['carbon_pools_path'])
carbon.execute(args)
# Add assertions for npv for the future scenario.
# The npv was calculated based on _calculate_npv in carbon.py.
# Ensure NPV calculations are correct.
# Valuation constant based on provided args is 59.00136.
# Alternate: 59.00136 * -58 Mg/ha = -3422.079 Mg/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_fut.tif'), -0.3422078)
os.path.join(args['workspace_dir'], 'npv_alt.tif'), -3422.079)
def test_carbon_missing_landcover_values(self):
"""Carbon: testing expected exception on missing LULC codes."""
@ -209,7 +255,7 @@ class CarbonTests(unittest.TestCase):
'n_workers': -1,
}
lulc_names = ['lulc_cur_path', 'lulc_fut_path']
lulc_names = ['lulc_bas_path', 'lulc_alt_path']
for fill_val, lulc_name in enumerate(lulc_names, 200):
args[lulc_name] = os.path.join(args['workspace_dir'],
lulc_name + '.tif')
@ -236,14 +282,14 @@ class CarbonTests(unittest.TestCase):
'do_valuation': True,
'price_per_metric_ton_of_c': 43.0,
'rate_change': 2.8,
'lulc_cur_year': 2016,
'lulc_fut_year': 2030,
'lulc_bas_year': 2016,
'lulc_alt_year': 2030,
'discount_rate': -7.1,
'n_workers': -1,
}
# Create LULC rasters and pools csv in workspace and add them to args.
lulc_names = ['lulc_cur_path', 'lulc_fut_path', 'lulc_redd_path']
lulc_names = ['lulc_bas_path', 'lulc_alt_path']
for fill_val, lulc_name in enumerate(lulc_names, 1):
args[lulc_name] = os.path.join(args['workspace_dir'],
lulc_name + '.tif')
@ -255,12 +301,11 @@ class CarbonTests(unittest.TestCase):
carbon.execute(args)
# Add assertions for npv for future and REDD scenarios.
# The npv was calculated based on _calculate_npv in carbon.py.
# Ensure NPV calculations are correct.
# Valuation constant based on provided args is 59.00136.
# Alternate: 59.00136 * -58 Mg/ha = -3422.079 Mg/ha
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_fut.tif'), -0.3422078)
assert_raster_equal_value(
os.path.join(args['workspace_dir'], 'npv_redd.tif'), -0.4602106)
os.path.join(args['workspace_dir'], 'npv_alt.tif'), -3422.079)
def test_generate_carbon_map(self):
"""Test `_generate_carbon_map`"""
@ -303,11 +348,10 @@ class CarbonTests(unittest.TestCase):
out_carbon_stock_path)
# open output carbon stock raster and check values
actual_carbon_stock = gdal.Open(out_carbon_stock_path)
band = actual_carbon_stock.GetRasterBand(1)
actual_carbon_stock = band.ReadAsArray()
actual_carbon_stock = pygeoprocessing.raster_to_numpy_array(
out_carbon_stock_path)
expected_carbon_stock = numpy.array([[0.5, 0.5], [0.006, 0.012]],
expected_carbon_stock = numpy.array([[5000, 5000], [60, 120]],
dtype=numpy.float32)
numpy.testing.assert_array_equal(actual_carbon_stock,
@ -317,8 +361,8 @@ class CarbonTests(unittest.TestCase):
"""Test `_calculate_valuation_constant`"""
from natcap.invest.carbon import _calculate_valuation_constant
valuation_constant = _calculate_valuation_constant(lulc_cur_year=2010,
lulc_fut_year=2012,
valuation_constant = _calculate_valuation_constant(lulc_bas_year=2010,
lulc_alt_year=2012,
discount_rate=50,
rate_change=5,
price_per_metric_ton_of_c=50)
@ -334,7 +378,7 @@ class CarbonValidationTests(unittest.TestCase):
self.workspace_dir = tempfile.mkdtemp()
self.base_required_keys = [
'workspace_dir',
'lulc_cur_path',
'lulc_bas_path',
'carbon_pools_path',
]
@ -362,21 +406,7 @@ class CarbonValidationTests(unittest.TestCase):
invalid_keys = validation.get_invalid_keys(validation_errors)
expected_missing_keys = set(
self.base_required_keys +
['lulc_fut_path'])
self.assertEqual(invalid_keys, expected_missing_keys)
def test_missing_keys_redd(self):
"""Carbon Validate: assert missing do_redd keys."""
from natcap.invest import carbon
from natcap.invest import validation
args = {'do_redd': True}
validation_errors = carbon.validate(args)
invalid_keys = validation.get_invalid_keys(validation_errors)
expected_missing_keys = set(
self.base_required_keys +
['calc_sequestration',
'lulc_redd_path'])
['lulc_alt_path'])
self.assertEqual(invalid_keys, expected_missing_keys)
def test_missing_keys_valuation(self):
@ -393,6 +423,20 @@ class CarbonValidationTests(unittest.TestCase):
'price_per_metric_ton_of_c',
'discount_rate',
'rate_change',
'lulc_cur_year',
'lulc_fut_year'])
'lulc_bas_year',
'lulc_alt_year'])
self.assertEqual(invalid_keys, expected_missing_keys)
def test_invalid_lulc_years(self):
"""Test Alternate LULC year < Baseline LULC year raises a ValueError"""
from natcap.invest import carbon
args = {
'workspace_dir': self.workspace_dir,
'do_valuation': True,
'lulc_bas_year': 2025,
'lulc_alt_year': 2023,
}
with self.assertRaises(ValueError):
carbon.execute(args)

2
tests/test_cli.py
View File

@ -221,7 +221,7 @@ class CLIHeadlessTests(unittest.TestCase):
validation_output = stdout_stream.getvalue()
# it's expected that these keys are missing because the only
# key we included was the workspace_dir
expected_warning = [(['carbon_pools_path', 'lulc_cur_path'],
expected_warning = [(['carbon_pools_path', 'lulc_bas_path'],
validation.MESSAGES['MISSING_KEY'])]
self.assertEqual(validation_output, str(expected_warning))
self.assertEqual(exit_cm.exception.code, 0)

8
tests/test_coastal_blue_carbon.py
View File

@ -98,7 +98,7 @@ class TestPreprocessor(unittest.TestCase):
with open(lulc_attributes_path, 'w') as attributes_table:
attributes_table.write(textwrap.dedent(
"""\
lulc-class,code,is_coastal_blue_carbon_habitat
lulc-class,lucode,is_coastal_blue_carbon_habitat
mangrove,1,TRUE
parkinglot,2,FALSE
"""))
@ -177,7 +177,7 @@ class TestPreprocessor(unittest.TestCase):
expected_landcover_codes = set(range(0, 24))
found_landcover_codes = set(pandas.read_csv(
os.path.join(outputs_dir, 'carbon_biophysical_table_template_150225.csv')
)['code'].values)
)['lucode'].values)
self.assertEqual(expected_landcover_codes, found_landcover_codes)
def test_transition_table(self):
@ -207,7 +207,7 @@ class TestPreprocessor(unittest.TestCase):
landcover_table_path = os.path.join(self.workspace_dir,
'lulc_table.csv')
with open(landcover_table_path, 'w') as lulc_csv:
lulc_csv.write('code,lulc-class,is_coastal_blue_carbon_habitat\n')
lulc_csv.write('lucode,lulc-class,is_coastal_blue_carbon_habitat\n')
lulc_csv.write('0,mangrove,True\n')
lulc_csv.write('1,parking lot,False\n')
@ -451,7 +451,7 @@ class TestCBC2(unittest.TestCase):
wkt = srs.ExportToWkt()
biophysical_table = [
['code', 'lulc-class', 'biomass-initial', 'soil-initial',
['lucode', 'lulc-class', 'biomass-initial', 'soil-initial',
'litter-initial', 'biomass-half-life',
'biomass-low-impact-disturb', 'biomass-med-impact-disturb',
'biomass-high-impact-disturb', 'biomass-yearly-accumulation',

405
tests/test_crop_production.py
View File

@ -22,6 +22,20 @@ TEST_DATA_PATH = os.path.join(
'crop_production_model')
def _get_pixels_per_hectare(raster_path):
"""Calculate number of pixels per hectare for a given raster.
Args:
raster_path (str): full path to the raster.
Returns:
A float representing the number of pixels per hectare.
"""
raster_info = pygeoprocessing.get_raster_info(raster_path)
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
return 10000 / pixel_area
def make_aggregate_vector(path_to_shp):
"""
Generate shapefile with two overlapping polygons
@ -203,6 +217,41 @@ class CropProductionTests(unittest.TestCase):
pandas.testing.assert_frame_equal(
expected_result_table, result_table, check_dtype=False)
# Check raster outputs to make sure values are in Mg/ha.
# Raster sum is (Mg•px)/(ha•yr).
# Result table reports totals in Mg/yr.
# To convert from Mg/yr to (Mg•px)/(ha•yr), multiply by px/ha.
expected_raster_sums = {}
for (index, crop) in [(0, 'barley'), (1, 'soybean'), (2, 'wheat')]:
filename = crop + '_observed_production.tif'
pixels_per_hectare = _get_pixels_per_hectare(
os.path.join(args['workspace_dir'], filename))
expected_raster_sums[filename] = (
expected_result_table.loc[index]['production_observed']
* pixels_per_hectare)
for percentile in ['25', '50', '75', '95']:
filename = (
crop + '_yield_' + percentile + 'th_production.tif')
col_name = 'production_' + percentile + 'th'
pixels_per_hectare = _get_pixels_per_hectare(
os.path.join(args['workspace_dir'], filename))
expected_raster_sums[filename] = (
expected_result_table.loc[index][col_name]
* pixels_per_hectare)
for filename in expected_raster_sums:
raster_path = os.path.join(args['workspace_dir'], filename)
raster_info = pygeoprocessing.get_raster_info(raster_path)
nodata = raster_info['nodata'][0]
raster_sum = 0.0
for _, block in pygeoprocessing.iterblocks((raster_path, 1)):
raster_sum += numpy.sum(
block[~pygeoprocessing.array_equals_nodata(
block, nodata)], dtype=numpy.float32)
expected_sum = expected_raster_sums[filename]
numpy.testing.assert_allclose(raster_sum, expected_sum,
rtol=0, atol=0.1)
def test_crop_production_percentile_no_nodata(self):
"""Crop Production: test percentile model with undefined nodata raster.
@ -351,9 +400,6 @@ class CropProductionTests(unittest.TestCase):
'model_data_path': MODEL_DATA_PATH,
'fertilization_rate_table_path': os.path.join(
SAMPLE_DATA_PATH, 'crop_fertilization_rates.csv'),
'nitrogen_fertilization_rate': 29.6,
'phosphorus_fertilization_rate': 8.4,
'potassium_fertilization_rate': 14.2,
'n_workers': '-1'
}
@ -382,9 +428,6 @@ class CropProductionTests(unittest.TestCase):
'model_data_path': MODEL_DATA_PATH,
'fertilization_rate_table_path': os.path.join(
SAMPLE_DATA_PATH, 'crop_fertilization_rates.csv'),
'nitrogen_fertilization_rate': 29.6,
'phosphorus_fertilization_rate': 8.4,
'potassium_fertilization_rate': 14.2,
'n_workers': '-1'
}
@ -436,15 +479,13 @@ class CropProductionTests(unittest.TestCase):
'model_data_path': MODEL_DATA_PATH,
'fertilization_rate_table_path': os.path.join(
SAMPLE_DATA_PATH, 'crop_fertilization_rates.csv'),
'nitrogen_fertilization_rate': 29.6,
'phosphorus_fertilization_rate': 8.4,
'potassium_fertilization_rate': 14.2,
}
crop_production_regression.execute(args)
expected_agg_result_table = pandas.read_csv(
os.path.join(TEST_DATA_PATH, 'expected_regression_aggregate_results.csv'))
os.path.join(TEST_DATA_PATH,
'expected_regression_aggregate_results.csv'))
agg_result_table = pandas.read_csv(
os.path.join(args['workspace_dir'], 'aggregate_results.csv'))
pandas.testing.assert_frame_equal(
@ -462,6 +503,38 @@ class CropProductionTests(unittest.TestCase):
pandas.testing.assert_frame_equal(
expected_result_table, result_table, check_dtype=False)
# Check raster outputs to make sure values are in Mg/ha.
# Raster sum is (Mg•px)/(ha•yr).
# Result table reports totals in Mg/yr.
# To convert from Mg/yr to (Mg•px)/(ha•yr), multiply by px/ha.
expected_raster_sums = {}
for (index, crop) in [(0, 'barley'), (1, 'soybean'), (2, 'wheat')]:
filename = crop + '_observed_production.tif'
pixels_per_hectare = _get_pixels_per_hectare(
os.path.join(args['workspace_dir'], filename))
expected_raster_sums[filename] = (
expected_result_table.loc[index]['production_observed']
* pixels_per_hectare)
filename = crop + '_regression_production.tif'
pixels_per_hectare = _get_pixels_per_hectare(
os.path.join(args['workspace_dir'], filename))
expected_raster_sums[filename] = (
expected_result_table.loc[index]['production_modeled']
* pixels_per_hectare)
for filename in expected_raster_sums:
raster_path = os.path.join(args['workspace_dir'], filename)
raster_info = pygeoprocessing.get_raster_info(raster_path)
nodata = raster_info['nodata'][0]
raster_sum = 0.0
for _, block in pygeoprocessing.iterblocks((raster_path, 1)):
raster_sum += numpy.sum(
block[~pygeoprocessing.array_equals_nodata(
block, nodata)], dtype=numpy.float32)
expected_sum = expected_raster_sums[filename]
numpy.testing.assert_allclose(raster_sum, expected_sum,
rtol=0, atol=0.001)
def test_crop_production_regression_no_nodata(self):
"""Crop Production: test regression model with undefined nodata raster.
@ -480,9 +553,6 @@ class CropProductionTests(unittest.TestCase):
'model_data_path': MODEL_DATA_PATH,
'fertilization_rate_table_path': os.path.join(
SAMPLE_DATA_PATH, 'crop_fertilization_rates.csv'),
'nitrogen_fertilization_rate': 29.6,
'phosphorus_fertilization_rate': 8.4,
'potassium_fertilization_rate': 14.2,
}
# Create a raster based on the test data geotransform, but smaller and
@ -531,12 +601,11 @@ class CropProductionTests(unittest.TestCase):
lulc_array = numpy.array([[3, 3, 2], [3, -1, 3]])
fert_rate = 0.6
crop_lucode = 3
pixel_area_ha = 10
actual_result = _x_yield_op(y_max, b_x, c_x, lulc_array, fert_rate,
crop_lucode, pixel_area_ha)
expected_result = numpy.array([[-1, -19.393047, -1],
[26.776089, -1, 15.1231]])
crop_lucode)
expected_result = numpy.array([[-1, -1.9393047, -1],
[2.6776089, -1, 1.51231]])
numpy.testing.assert_allclose(actual_result, expected_result)
@ -571,13 +640,12 @@ class CropProductionTests(unittest.TestCase):
landcover_nodata = -9999
crop_lucode = 3
pixel_area_ha = 10
actual_result = _mask_observed_yield_op(
lulc_array, observed_yield_array, observed_yield_nodata,
landcover_nodata, crop_lucode, pixel_area_ha)
landcover_nodata, crop_lucode)
expected_result = numpy.array([[-10, 0, -1], [80, -99990, 0]])
expected_result = numpy.array([[-1, 0, -1], [8, -9999, 0]])
numpy.testing.assert_allclose(actual_result, expected_result)
@ -590,75 +658,75 @@ class CropProductionTests(unittest.TestCase):
"""Creates the expected results DataFrame."""
return pandas.DataFrame([
{'crop': 'corn', 'area (ha)': 20.0,
'production_observed': 8.0, 'production_modeled': 4.0,
'protein_modeled': 1562400.0, 'protein_observed': 3124800.0,
'lipid_modeled': 297600.0, 'lipid_observed': 595200.0,
'energy_modeled': 17707200.0, 'energy_observed': 35414400.0,
'ca_modeled': 1004400.0, 'ca_observed': 2008800.0,
'fe_modeled': 584040.0, 'fe_observed': 1168080.0,
'mg_modeled': 10416000.0, 'mg_observed': 20832000.0,
'ph_modeled': 26188800.0, 'ph_observed': 52377600.0,
'k_modeled': 64244400.0, 'k_observed': 128488800.0,
'na_modeled': 74400.0, 'na_observed': 148800.0,
'zn_modeled': 182280.0, 'zn_observed': 364560.0,
'cu_modeled': 70680.0, 'cu_observed': 141360.0,
'fl_modeled': 297600.0, 'fl_observed': 595200.0,
'mn_modeled': 107880.0, 'mn_observed': 215760.0,
'se_modeled': 3720.0, 'se_observed': 7440.0,
'vita_modeled': 111600.0, 'vita_observed': 223200.0,
'betac_modeled': 595200.0, 'betac_observed': 1190400.0,
'alphac_modeled': 85560.0, 'alphac_observed': 171120.0,
'vite_modeled': 29760.0, 'vite_observed': 59520.0,
'crypto_modeled': 59520.0, 'crypto_observed': 119040.0,
'lycopene_modeled': 13392.0, 'lycopene_observed': 26784.0,
'lutein_modeled': 2343600.0, 'lutein_observed': 4687200.0,
'betat_modeled': 18600.0, 'betat_observed': 37200.0,
'gammat_modeled': 78120.0, 'gammat_observed': 156240.0,
'deltat_modeled': 70680.0, 'deltat_observed': 141360.0,
'vitc_modeled': 252960.0, 'vitc_observed': 505920.0,
'thiamin_modeled': 14880.0, 'thiamin_observed': 29760.0,
'riboflavin_modeled': 66960.0, 'riboflavin_observed': 133920.0,
'niacin_modeled': 305040.0, 'niacin_observed': 610080.0,
'pantothenic_modeled': 33480.0, 'pantothenic_observed': 66960.0,
'vitb6_modeled': 52080.0, 'vitb6_observed': 104160.0,
'folate_modeled': 14322000.0, 'folate_observed': 28644000.0,
'vitb12_modeled': 74400.0, 'vitb12_observed': 148800.0,
'vitk_modeled': 1525200.0, 'vitk_observed': 3050400.0},
'production_observed': 80.0, 'production_modeled': 40.0,
'protein_modeled': 15624000.0, 'protein_observed': 31248000.0,
'lipid_modeled': 2976000.0, 'lipid_observed': 5952000.0,
'energy_modeled': 177072000.0, 'energy_observed': 354144000.0,
'ca_modeled': 10044000.0, 'ca_observed': 20088000.0,
'fe_modeled': 5840400.0, 'fe_observed': 11680800.0,
'mg_modeled': 104160000.0, 'mg_observed': 208320000.0,
'ph_modeled': 261888000.0, 'ph_observed': 523776000.0,
'k_modeled': 642444000.0, 'k_observed': 1284888000.0,
'na_modeled': 744000.0, 'na_observed': 1488000.0,
'zn_modeled': 1822800.0, 'zn_observed': 3645600.0,
'cu_modeled': 706800.0, 'cu_observed': 1413600.0,
'fl_modeled': 2976000.0, 'fl_observed': 5952000.0,
'mn_modeled': 1078800.0, 'mn_observed': 2157600.0,
'se_modeled': 37200.0, 'se_observed': 74400.0,
'vita_modeled': 1116000.0, 'vita_observed': 2232000.0,
'betac_modeled': 5952000.0, 'betac_observed': 11904000.0,
'alphac_modeled': 855600.0, 'alphac_observed': 1711200.0,
'vite_modeled': 297600.0, 'vite_observed': 595200.0,
'crypto_modeled': 595200.0, 'crypto_observed': 1190400.0,
'lycopene_modeled': 133920.0, 'lycopene_observed': 267840.0,
'lutein_modeled': 23436000.0, 'lutein_observed': 46872000.0,
'betat_modeled': 186000.0, 'betat_observed': 372000.0,
'gammat_modeled': 781200.0, 'gammat_observed': 1562400.0,
'deltat_modeled': 706800.0, 'deltat_observed': 1413600.0,
'vitc_modeled': 2529600.0, 'vitc_observed': 5059200.0,
'thiamin_modeled': 148800.0, 'thiamin_observed': 297600.0,
'riboflavin_modeled': 669600.0, 'riboflavin_observed': 1339200.0,
'niacin_modeled': 3050400.0, 'niacin_observed': 6100800.0,
'pantothenic_modeled': 334800.0, 'pantothenic_observed': 669600.0,
'vitb6_modeled': 520800.0, 'vitb6_observed': 1041600.0,
'folate_modeled': 143220000.0, 'folate_observed': 286440000.0,
'vitb12_modeled': 744000.0, 'vitb12_observed': 1488000.0,
'vitk_modeled': 15252000.0, 'vitk_observed': 30504000.0},
{'crop': 'soybean', 'area (ha)': 40.0,
'production_observed': 12.0, 'production_modeled': 7.0,
'protein_modeled': 2102100.0, 'protein_observed': 3603600.0,
'lipid_modeled': 127400.0, 'lipid_observed': 218400.0,
'energy_modeled': 6306300.0, 'energy_observed': 10810800.0,
'ca_modeled': 16370900.0, 'ca_observed': 28064400.0,
'fe_modeled': 1000090.0, 'fe_observed': 1714440.0,
'mg_modeled': 17836000.0, 'mg_observed': 30576000.0,
'ph_modeled': 44844800.0, 'ph_observed': 76876800.0,
'k_modeled': 12548900.0, 'k_observed': 21512400.0,
'na_modeled': 127400.0, 'na_observed': 218400.0,
'zn_modeled': 312130.0, 'zn_observed': 535080.0,
'cu_modeled': 101920.0, 'cu_observed': 174720.0,
'fl_modeled': 191100.0, 'fl_observed': 327600.0,
'mn_modeled': 331240.0, 'mn_observed': 567840.0,
'se_modeled': 19110.0, 'se_observed': 32760.0,
'vita_modeled': 191100.0, 'vita_observed': 327600.0,
'betac_modeled': 1019200.0, 'betac_observed': 1747200.0,
'alphac_modeled': 63700.0, 'alphac_observed': 109200.0,
'vite_modeled': 50960.0, 'vite_observed': 87360.0,
'crypto_modeled': 38220.0, 'crypto_observed': 65520.0,
'lycopene_modeled': 19110.0, 'lycopene_observed': 32760.0,
'lutein_modeled': 3885700.0, 'lutein_observed': 6661200.0,
'betat_modeled': 31850.0, 'betat_observed': 54600.0,
'gammat_modeled': 146510.0, 'gammat_observed': 251160.0,
'deltat_modeled': 76440.0, 'deltat_observed': 131040.0,
'vitc_modeled': 191100.0, 'vitc_observed': 327600.0,
'thiamin_modeled': 26754.0, 'thiamin_observed': 45864.0,
'riboflavin_modeled': 52234.0, 'riboflavin_observed': 89544.0,
'niacin_modeled': 777140.0, 'niacin_observed': 1332240.0,
'pantothenic_modeled': 58604.0, 'pantothenic_observed': 100464.0,
'vitb6_modeled': 343980.0, 'vitb6_observed': 589680.0,
'folate_modeled': 19428500.0, 'folate_observed': 33306000.0,
'vitb12_modeled': 191100.0, 'vitb12_observed': 327600.0,
'vitk_modeled': 2675400.0, 'vitk_observed': 4586400.0}])
'production_observed': 120.0, 'production_modeled': 70.0,
'protein_modeled': 21021000.0, 'protein_observed': 36036000.0,
'lipid_modeled': 1274000.0, 'lipid_observed': 2184000.0,
'energy_modeled': 63063000.0, 'energy_observed': 108108000.0,
'ca_modeled': 163709000.0, 'ca_observed': 280644000.0,
'fe_modeled': 10000900.0, 'fe_observed': 17144400.0,
'mg_modeled': 178360000.0, 'mg_observed': 305760000.0,
'ph_modeled': 448448000.0, 'ph_observed': 768768000.0,
'k_modeled': 125489000.0, 'k_observed': 215124000.0,
'na_modeled': 1274000.0, 'na_observed': 2184000.0,
'zn_modeled': 3121300.0, 'zn_observed': 5350800.0,
'cu_modeled': 1019200.0, 'cu_observed': 1747200.0,
'fl_modeled': 1911000.0, 'fl_observed': 3276000.0,
'mn_modeled': 3312400.0, 'mn_observed': 5678400.0,
'se_modeled': 191100.0, 'se_observed': 327600.0,
'vita_modeled': 1911000.0, 'vita_observed': 3276000.0,
'betac_modeled': 10192000.0, 'betac_observed': 17472000.0,
'alphac_modeled': 637000.0, 'alphac_observed': 1092000.0,
'vite_modeled': 509600.0, 'vite_observed': 873600.0,
'crypto_modeled': 382200.0, 'crypto_observed': 655200.0,
'lycopene_modeled': 191100.0, 'lycopene_observed': 327600.0,
'lutein_modeled': 38857000.0, 'lutein_observed': 66612000.0,
'betat_modeled': 318500.0, 'betat_observed': 546000.0,
'gammat_modeled': 1465100.0, 'gammat_observed': 2511600.0,
'deltat_modeled': 764400.0, 'deltat_observed': 1310400.0,
'vitc_modeled': 1911000.0, 'vitc_observed': 3276000.0,
'thiamin_modeled': 267540.0, 'thiamin_observed': 458640.0,
'riboflavin_modeled': 522340.0, 'riboflavin_observed': 895440.0,
'niacin_modeled': 7771400.0, 'niacin_observed': 13322400.0,
'pantothenic_modeled': 586040.0, 'pantothenic_observed': 1004640.0,
'vitb6_modeled': 3439800.0, 'vitb6_observed': 5896800.0,
'folate_modeled': 194285000.0, 'folate_observed': 333060000.0,
'vitb12_modeled': 1911000.0, 'vitb12_observed': 3276000.0,
'vitk_modeled': 26754000.0, 'vitk_observed': 45864000.0}])
nutrient_df = create_nutrient_df()
@ -702,76 +770,76 @@ class CropProductionTests(unittest.TestCase):
"""Create expected output results"""
# Define the new values manually
return pandas.DataFrame([
{"FID": 0, "corn_modeled": 1, "corn_observed": 4,
"soybean_modeled": 2, "soybean_observed": 5,
"protein_modeled": 991200, "protein_observed": 3063900,
"lipid_modeled": 110800, "lipid_observed": 388600,
"energy_modeled": 6228600, "energy_observed": 22211700,
"ca_modeled": 4928500, "ca_observed": 12697900,
"fe_modeled": 431750, "fe_observed": 1298390,
"mg_modeled": 7700000, "mg_observed": 23156000,
"ph_modeled": 19360000, "ph_observed": 58220800,
"k_modeled": 19646500, "k_observed": 73207900,
"na_modeled": 55000, "na_observed": 165400,
"zn_modeled": 134750, "zn_observed": 405230,
"cu_modeled": 46790, "cu_observed": 143480,
"fl_modeled": 129000, "fl_observed": 434100,
"mn_modeled": 121610, "mn_observed": 344480,
"se_modeled": 6390, "se_observed": 17370,
"vita_modeled": 82500, "vita_observed": 248100,
"betac_modeled": 440000, "betac_observed": 1323200,
"alphac_modeled": 39590, "alphac_observed": 131060,
"vite_modeled": 22000, "vite_observed": 66160,
"crypto_modeled": 25800, "crypto_observed": 86820,
"lycopene_modeled": 8808, "lycopene_observed": 27042,
"lutein_modeled": 1696100, "lutein_observed": 5119100,
"betat_modeled": 13750, "betat_observed": 41350,
"gammat_modeled": 61390, "gammat_observed": 182770,
"deltat_modeled": 39510, "deltat_observed": 125280,
"vitc_modeled": 117840, "vitc_observed": 389460,
"thiamin_modeled": 11364, "thiamin_observed": 33990,
"riboflavin_modeled": 31664, "riboflavin_observed": 104270,
"niacin_modeled": 298300, "niacin_observed": 860140,
"pantothenic_modeled": 25114, "pantothenic_observed": 75340,
"vitb6_modeled": 111300, "vitb6_observed": 297780,
"folate_modeled": 9131500, "folate_observed": 28199500,
"vitb12_modeled": 73200, "vitb12_observed": 210900,
"vitk_modeled": 1145700, "vitk_observed": 3436200},
{"FID": 1, "corn_modeled": 4, "corn_observed": 8,
"soybean_modeled": 7, "soybean_observed": 12,
"protein_modeled": 3664500, "protein_observed": 6728400,
"lipid_modeled": 425000, "lipid_observed": 813600,
"energy_modeled": 24013500, "energy_observed": 46225200,
"ca_modeled": 17375300, "ca_observed": 30073200,
"fe_modeled": 1584130, "fe_observed": 2882520,
"mg_modeled": 28252000, "mg_observed": 51408000,
"ph_modeled": 71033600, "ph_observed": 129254400,
"k_modeled": 76793300, "k_observed": 150001200,
"na_modeled": 201800, "na_observed": 367200,
"zn_modeled": 494410, "zn_observed": 899640,
"cu_modeled": 172600, "cu_observed": 316080,
"fl_modeled": 488700, "fl_observed": 922800,
"mn_modeled": 439120, "mn_observed": 783600,
"se_modeled": 22830, "se_observed": 40200,
"vita_modeled": 302700, "vita_observed": 550800,
"betac_modeled": 1614400, "betac_observed": 2937600,
"alphac_modeled": 149260, "alphac_observed": 280320,
"vite_modeled": 80720, "vite_observed": 146880,
"crypto_modeled": 97740, "crypto_observed": 184560,
"lycopene_modeled": 32502, "lycopene_observed": 59544,
"lutein_modeled": 6229300, "lutein_observed": 11348400,
"betat_modeled": 50450, "betat_observed": 91800,
"gammat_modeled": 224630, "gammat_observed": 407400,
"deltat_modeled": 147120, "deltat_observed": 272400,
"vitc_modeled": 444060, "vitc_observed": 833520,
"thiamin_modeled": 41634, "thiamin_observed": 75624,
"riboflavin_modeled": 119194, "riboflavin_observed": 223464,
"niacin_modeled": 1082180, "niacin_observed": 1942320,
"pantothenic_modeled": 92084, "pantothenic_observed": 167424,
"vitb6_modeled": 396060, "vitb6_observed": 693840,
"folate_modeled": 33750500, "folate_observed": 61950000,
"vitb12_modeled": 265500, "vitb12_observed": 476400,
"vitk_modeled": 4200600, "vitk_observed": 7636800}
{"FID": 0, "corn_modeled": 10, "corn_observed": 40,
"soybean_modeled": 20, "soybean_observed": 50,
"protein_modeled": 9912000, "protein_observed": 30639000,
"lipid_modeled": 1108000, "lipid_observed": 3886000,
"energy_modeled": 62286000, "energy_observed": 222117000,
"ca_modeled": 49285000, "ca_observed": 126979000,
"fe_modeled": 4317500, "fe_observed": 12983900,
"mg_modeled": 77000000, "mg_observed": 231560000,
"ph_modeled": 193600000, "ph_observed": 582208000,
"k_modeled": 196465000, "k_observed": 732079000,
"na_modeled": 550000, "na_observed": 1654000,
"zn_modeled": 1347500, "zn_observed": 4052300,
"cu_modeled": 467900, "cu_observed": 1434800,
"fl_modeled": 1290000, "fl_observed": 4341000,
"mn_modeled": 1216100, "mn_observed": 3444800,
"se_modeled": 63900, "se_observed": 173700,
"vita_modeled": 825000, "vita_observed": 2481000,
"betac_modeled": 4400000, "betac_observed": 13232000,
"alphac_modeled": 395900, "alphac_observed": 1310600,
"vite_modeled": 220000, "vite_observed": 661600,
"crypto_modeled": 258000, "crypto_observed": 868200,
"lycopene_modeled": 88080, "lycopene_observed": 270420,
"lutein_modeled": 16961000, "lutein_observed": 51191000,
"betat_modeled": 137500, "betat_observed": 413500,
"gammat_modeled": 613900, "gammat_observed": 1827700,
"deltat_modeled": 395100, "deltat_observed": 1252800,
"vitc_modeled": 1178400, "vitc_observed": 3894600,
"thiamin_modeled": 113640, "thiamin_observed": 339900,
"riboflavin_modeled": 316640, "riboflavin_observed": 1042700,
"niacin_modeled": 2983000, "niacin_observed": 8601400,
"pantothenic_modeled": 251140, "pantothenic_observed": 753400,
"vitb6_modeled": 1113000, "vitb6_observed": 2977800,
"folate_modeled": 91315000, "folate_observed": 281995000,
"vitb12_modeled": 732000, "vitb12_observed": 2109000,
"vitk_modeled": 11457000, "vitk_observed": 34362000},
{"FID": 1, "corn_modeled": 40, "corn_observed": 80,
"soybean_modeled": 70, "soybean_observed": 120,
"protein_modeled": 36645000, "protein_observed": 67284000,
"lipid_modeled": 4250000, "lipid_observed": 8136000,
"energy_modeled": 240135000, "energy_observed": 462252000,
"ca_modeled": 173753000, "ca_observed": 300732000,
"fe_modeled": 15841300, "fe_observed": 28825200,
"mg_modeled": 282520000, "mg_observed": 514080000,
"ph_modeled": 710336000, "ph_observed": 1292544000,
"k_modeled": 767933000, "k_observed": 1500012000,
"na_modeled": 2018000, "na_observed": 3672000,
"zn_modeled": 4944100, "zn_observed": 8996400,
"cu_modeled": 1726000, "cu_observed": 3160800,
"fl_modeled": 4887000, "fl_observed": 9228000,
"mn_modeled": 4391200, "mn_observed": 7836000,
"se_modeled": 228300, "se_observed": 402000,
"vita_modeled": 3027000, "vita_observed": 5508000,
"betac_modeled": 16144000, "betac_observed": 29376000,
"alphac_modeled": 1492600, "alphac_observed": 2803200,
"vite_modeled": 807200, "vite_observed": 1468800,
"crypto_modeled": 977400, "crypto_observed": 1845600,
"lycopene_modeled": 325020, "lycopene_observed": 595440,
"lutein_modeled": 62293000, "lutein_observed": 113484000,
"betat_modeled": 504500, "betat_observed": 918000,
"gammat_modeled": 2246300, "gammat_observed": 4074000,
"deltat_modeled": 1471200, "deltat_observed": 2724000,
"vitc_modeled": 4440600, "vitc_observed": 8335200,
"thiamin_modeled": 416340, "thiamin_observed": 756240,
"riboflavin_modeled": 1191940, "riboflavin_observed": 2234640,
"niacin_modeled": 10821800, "niacin_observed": 19423200,
"pantothenic_modeled": 920840, "pantothenic_observed": 1674240,
"vitb6_modeled": 3960600, "vitb6_observed": 6938400,
"folate_modeled": 337505000, "folate_observed": 619500000,
"vitb12_modeled": 2655000, "vitb12_observed": 4764000,
"vitk_modeled": 42006000, "vitk_observed": 76368000}
], dtype=float)
workspace = self.workspace_dir
@ -792,21 +860,21 @@ class CropProductionTests(unittest.TestCase):
output_dir = os.path.join(workspace, "OUTPUT")
os.makedirs(output_dir, exist_ok=True)
file_suffix = 'test'
target_aggregate_table_path = '' # unused
_AGGREGATE_TABLE_FILE_PATTERN = os.path.join(
'.', 'aggregate_results%s.csv')
aggregate_table_path = os.path.join(
output_dir, _AGGREGATE_TABLE_FILE_PATTERN % file_suffix)
pixel_area_ha = 10
_create_crop_rasters(output_dir, crop_names, file_suffix)
aggregate_regression_results_to_polygons(
base_aggregate_vector_path, target_aggregate_vector_path,
landcover_raster_projection, crop_names,
nutrient_df, output_dir, file_suffix,
target_aggregate_table_path)
_AGGREGATE_TABLE_FILE_PATTERN = os.path.join(
'.','aggregate_results%s.csv')
aggregate_table_path = os.path.join(
output_dir, _AGGREGATE_TABLE_FILE_PATTERN % file_suffix)
aggregate_table_path, landcover_raster_projection, crop_names,
nutrient_df, pixel_area_ha, output_dir, file_suffix)
actual_aggregate_table = pandas.read_csv(aggregate_table_path,
dtype=float)
@ -1053,7 +1121,6 @@ class CropProductionTests(unittest.TestCase):
pandas.testing.assert_frame_equal(actual_table, expected_table,
check_dtype=False)
class CropValidationTests(unittest.TestCase):
"""Tests for the Crop Productions' MODEL_SPEC and validation."""

54
tests/test_forest_carbon_edge.py
View File

@ -11,6 +11,8 @@ from shapely.geometry import Polygon
import pygeoprocessing
import pickle
import pygeoprocessing
gdal.UseExceptions()
REGRESSION_DATA = os.path.join(
os.path.dirname(__file__), '..', 'data', 'invest-test-data',
@ -112,20 +114,31 @@ class ForestCarbonEdgeTests(unittest.TestCase):
args['workspace_dir'])
self._assert_vector_results_close(
args['workspace_dir'], 'id', ['c_sum', 'c_ha_mean'], os.path.join(
'id',
['c_sum', 'c_ha_mean'],
os.path.join(
args['workspace_dir'], 'aggregated_carbon_stocks.shp'),
os.path.join(REGRESSION_DATA, 'agg_results_base.shp'))
expected_carbon_raster = gdal.OpenEx(os.path.join(REGRESSION_DATA,
'carbon_map.tif'))
expected_carbon_band = expected_carbon_raster.GetRasterBand(1)
expected_carbon_array = expected_carbon_band.ReadAsArray()
actual_carbon_raster = gdal.OpenEx(os.path.join(REGRESSION_DATA,
'carbon_map.tif'))
actual_carbon_band = actual_carbon_raster.GetRasterBand(1)
actual_carbon_array = actual_carbon_band.ReadAsArray()
self.assertTrue(numpy.allclose(expected_carbon_array,
actual_carbon_array))
# Check raster output to make sure values are in Mg/ha.
raster_path = os.path.join(args['workspace_dir'], 'carbon_map.tif')
raster_info = pygeoprocessing.get_raster_info(raster_path)
nodata = raster_info['nodata'][0]
raster_sum = 0.0
for _, block in pygeoprocessing.iterblocks((raster_path, 1)):
raster_sum += numpy.sum(
block[~pygeoprocessing.array_equals_nodata(
block, nodata)], dtype=numpy.float64)
# expected_sum_per_pixel_values is in Mg, calculated from raster
# generated by the model when each pixel value was in Mg/px.
# Since pixel values are now Mg/ha, raster sum is (Mg•px)/ha.
# To convert expected_sum_per_pixel_values from Mg, multiply by px/ha.
expected_sum_per_pixel_values = 21414391.997192383
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
pixels_per_hectare = 10000 / pixel_area
expected_sum = expected_sum_per_pixel_values * pixels_per_hectare
numpy.testing.assert_allclose(raster_sum, expected_sum)
def test_carbon_dup_output(self):
"""Forest Carbon Edge: test for existing output overlap."""
@ -186,7 +199,8 @@ class ForestCarbonEdgeTests(unittest.TestCase):
REGRESSION_DATA, 'file_list_no_edge_effect.txt'),
args['workspace_dir'])
self._assert_vector_results_close(
args['workspace_dir'], 'id', ['c_sum', 'c_ha_mean'],
'id',
['c_sum', 'c_ha_mean'],
os.path.join(
args['workspace_dir'],
'aggregated_carbon_stocks_small_no_edge_effect.shp'),
@ -220,7 +234,7 @@ class ForestCarbonEdgeTests(unittest.TestCase):
expected_message = 'Could not interpret carbon pool value'
actual_message = str(cm.exception)
self.assertTrue(expected_message in actual_message, actual_message)
def test_missing_lulc_value(self):
"""Forest Carbon Edge: test with missing LULC value."""
from natcap.invest import forest_carbon_edge_effect
@ -325,7 +339,7 @@ class ForestCarbonEdgeTests(unittest.TestCase):
gdal.OF_VECTOR | gdal.GA_Update) as ws_ds:
ws_layer = ws_ds.GetLayer()
for field_name, expected_value in zip(['c_sum', 'c_ha_mean'],
[21., 3.36]):
[0.0021, 0.000336]):
actual_values = [ws_feat.GetField(field_name)
for ws_feat in ws_layer][0]
error_msg = f"Error with {field_name} in agg_carbon.shp"
@ -361,7 +375,8 @@ class ForestCarbonEdgeTests(unittest.TestCase):
carbon_map_path)
actual_output = pygeoprocessing.raster_to_numpy_array(carbon_map_path)
expected_output = numpy.array([[0.01, 0.05, 0.02], [0.02, 0.05, 0.01]])
expected_output = numpy.array([[100, 500, 200], [200, 500, 100]])
numpy.testing.assert_allclose(actual_output, expected_output)
def test_map_distance_from_tropical_forest_edge(self):
@ -505,9 +520,9 @@ class ForestCarbonEdgeTests(unittest.TestCase):
tropical_forest_edge_carbon_map_path)
expected_output = numpy.array(
[[-1, -1, -1, -1, -1, -1, -1],
[-1, 1.3486482, 1.3903661, 1.5450714, 1.6976272, 1.7436424, -1],
[-1, 1.7600988, 3.716307, 4.004815, 3.8613932, 2.2213786, -1],
[-1, 2.2157857, 2.2673838, 2.448313, 2.6430724, 2.6987493, -1],
[-1, 13486.482, 13903.661, 15450.714, 16976.272, 17436.426, -1],
[-1, 17600.988, 37163.07, 40048.15, 38613.932, 22213.786, -1],
[-1, 22157.857, 22673.838, 24483.13, 26430.724, 26987.493, -1],
[-1, -1, -1, -1, -1, -1, -1]])
numpy.testing.assert_allclose(actual_output, expected_output)
@ -544,12 +559,11 @@ class ForestCarbonEdgeTests(unittest.TestCase):
'\n'.join(missing_files))
def _assert_vector_results_close(
self, workspace_dir, id_fieldname, field_list, result_vector_path,
self, id_fieldname, field_list, result_vector_path,
expected_vector_path):
"""Test workspace state against expected aggregate results.
Args:
workspace_dir (string): path to the completed model workspace
id_fieldname (string): fieldname of the unique ID.
field_list (list of string): list of fields to check
near-equality.

305
tests/test_habitat_quality.py
View File

@ -144,14 +144,14 @@ def make_sensitivity_samp_csv(
"""
if include_threat:
with open(csv_path, 'w') as open_table:
open_table.write('LULC,NAME,HABITAT,threat_1,threat_2\n')
open_table.write('lucode,name,habitat,threat_1,threat_2\n')
open_table.write('1,"lulc 1",1,1,1\n')
if not missing_lines:
open_table.write('2,"lulc 2",0.5,0.5,1\n')
open_table.write('3,"lulc 3",0,0.3,1\n')
else:
with open(csv_path, 'w') as open_table:
open_table.write('LULC,NAME,HABITAT\n')
open_table.write('lucode,name,habitat\n')
open_table.write('1,"lulc 1",1\n')
if not missing_lines:
open_table.write('2,"lulc 2",0.5\n')
@ -236,9 +236,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -277,11 +277,11 @@ class HabitatQualityTests(unittest.TestCase):
with open(csv_path, newline='') as csvfile:
reader = csv.DictReader(csvfile, delimiter=',')
self.assertEqual(reader.fieldnames,
['lulc_code', 'rarity_value'])
['lucode', 'rarity_value'])
for (exp_lulc, exp_rarity,
places_to_round) in expected_csv_values[csv_filename]:
row = next(reader)
self.assertEqual(int(row['lulc_code']), exp_lulc)
self.assertEqual(int(row['lucode']), exp_lulc)
self.assertAlmostEqual(float(row['rarity_value']),
exp_rarity, places_to_round)
@ -344,9 +344,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_reprojected_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_reprojected_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -405,9 +405,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -467,9 +467,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -527,9 +527,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -581,9 +581,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,%s,linear,,1111_c.tif,1111_f.tif\n' % threatnames[0])
'40,0.7,%s,linear,,1111_c.tif,1111_f.tif\n' % threatnames[0])
open_table.write(
'0.07,1.0,%s,exponential,,2222_c.tif,2222_f.tif\n'
'70,1.0,%s,exponential,,2222_c.tif,2222_f.tif\n'
% threatnames[1])
args = {
@ -610,7 +610,7 @@ class HabitatQualityTests(unittest.TestCase):
with open(args['sensitivity_table_path'], 'w') as open_table:
open_table.write(
'LULC,NAME,HABITAT,%s,%s\n' % tuple(threatnames))
'lucode,name,habitat,%s,%s\n' % tuple(threatnames))
open_table.write('1,"lulc 1",1,1,1\n')
open_table.write('2,"lulc 2",0.5,0.5,1\n')
open_table.write('3,"lulc 3",0,0.3,1\n')
@ -665,9 +665,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(KeyError):
@ -709,7 +709,7 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.07,0.8,missing_threat,linear,,missing_threat_c.tif,'
'70,0.8,missing_threat,linear,,missing_threat_c.tif,'
'missing_threat_f.tif\n')
with self.assertRaises(ValueError):
@ -755,9 +755,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError):
@ -803,7 +803,7 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'0.0,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -848,7 +848,7 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,invalid,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,invalid,,threat_1_c.tif,threat_1_f.tif\n')
with self.assertRaises(ValueError):
habitat_quality.execute(args)
@ -892,9 +892,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,threat_2_c.tif,'
'70,1.0,threat_2,exponential,threat_2_c.tif,'
'threat_2_f.tif\n')
try:
@ -943,9 +943,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif\n')
'70,1.0,threat_2,exponential,,threat_2_c.tif\n')
try:
habitat_quality.execute(args)
@ -987,9 +987,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1031,9 +1031,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -1057,7 +1057,7 @@ class HabitatQualityTests(unittest.TestCase):
args['workspace_dir'], 'sensitivity_samp.csv')
with open(args['sensitivity_table_path'], 'w') as open_table:
open_table.write('Lulc,Name,habitat,Threat_1,THREAT_2\n')
open_table.write('LuCode,Name,habitat,Threat_1,THREAT_2\n')
open_table.write('1,"lulc 1",1,1,1\n')
open_table.write('2,"lulc 2",0.5,0.5,1\n')
open_table.write('3,"lulc 3",0,0.3,1\n')
@ -1079,9 +1079,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'Max_Dist,Weight,threat,Decay,BASE_PATH,cur_PATH,fut_path\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -1124,9 +1124,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
habitat_quality.execute(args)
@ -1190,9 +1190,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1262,9 +1262,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -1312,9 +1312,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
args['lulc_cur_path'], args['access_vector_path'] = (
@ -1370,9 +1370,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_3,exponential,,threat_2_c.tif,'
'70,1.0,threat_3,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
# At least one threat header is expected, so there should be a message
@ -1383,6 +1383,159 @@ class HabitatQualityTests(unittest.TestCase):
validate_result[0][1],
habitat_quality.MISSING_SENSITIVITY_TABLE_THREATS_MSG))
def test_habitat_quality_validation_invalid_max_dist(self):
"""Habitat Quality: test validation for max_dist <= 0."""
from natcap.invest import habitat_quality
from natcap.invest import utils
args = {
'half_saturation_constant': '0.5',
'results_suffix': 'regression',
'workspace_dir': self.workspace_dir,
'n_workers': -1,
}
args['access_vector_path'] = os.path.join(
args['workspace_dir'], 'access_samp.shp')
make_access_shp(args['access_vector_path'])
scenarios = ['_bas_', '_cur_', '_fut_']
for lulc_val, scenario in enumerate(scenarios, start=1):
lulc_array = numpy.ones((100, 100), dtype=numpy.int8)
lulc_array[50:, :] = lulc_val
args['lulc' + scenario + 'path'] = os.path.join(
args['workspace_dir'], 'lc_samp' + scenario + 'b.tif')
make_raster_from_array(
lulc_array, args['lulc' + scenario + 'path'])
args['sensitivity_table_path'] = os.path.join(
args['workspace_dir'], 'sensitivity_samp.csv')
make_sensitivity_samp_csv(args['sensitivity_table_path'])
make_threats_raster(args['workspace_dir'])
args['threats_table_path'] = os.path.join(
args['workspace_dir'], 'threats_samp.csv')
# create the threat CSV table
with open(args['threats_table_path'], 'w') as open_table:
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
self.assertEqual(len(validate_result), 1)
self.assertEqual(validate_result[0][0], ['threats_table_path'])
self.assertTrue(utils.matches_format_string(
validate_result[0][1],
habitat_quality.INVALID_MAX_DIST_MSG))
def test_habitat_quality_validation_missing_max_dist(self):
"""Habitat Quality: test validation for missing max_dist."""
from natcap.invest import habitat_quality
from natcap.invest import utils
args = {
'half_saturation_constant': '0.5',
'results_suffix': 'regression',
'workspace_dir': self.workspace_dir,
'n_workers': -1,
}
args['access_vector_path'] = os.path.join(
args['workspace_dir'], 'access_samp.shp')
make_access_shp(args['access_vector_path'])
scenarios = ['_bas_', '_cur_', '_fut_']
for lulc_val, scenario in enumerate(scenarios, start=1):
lulc_array = numpy.ones((100, 100), dtype=numpy.int8)
lulc_array[50:, :] = lulc_val
args['lulc' + scenario + 'path'] = os.path.join(
args['workspace_dir'], 'lc_samp' + scenario + 'b.tif')
make_raster_from_array(
lulc_array, args['lulc' + scenario + 'path'])
args['sensitivity_table_path'] = os.path.join(
args['workspace_dir'], 'sensitivity_samp.csv')
make_sensitivity_samp_csv(args['sensitivity_table_path'])
make_threats_raster(args['workspace_dir'])
args['threats_table_path'] = os.path.join(
args['workspace_dir'], 'threats_samp.csv')
# create the threat CSV table
with open(args['threats_table_path'], 'w') as open_table:
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
',0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
self.assertEqual(len(validate_result), 1)
self.assertEqual(validate_result[0][0], ['threats_table_path'])
self.assertTrue(utils.matches_format_string(
validate_result[0][1],
habitat_quality.MISSING_MAX_DIST_MSG))
def test_habitat_quality_validation_missing_weight(self):
"""Habitat Quality: test validation for missing weight."""
from natcap.invest import habitat_quality
from natcap.invest import utils
args = {
'half_saturation_constant': '0.5',
'results_suffix': 'regression',
'workspace_dir': self.workspace_dir,
'n_workers': -1,
}
args['access_vector_path'] = os.path.join(
args['workspace_dir'], 'access_samp.shp')
make_access_shp(args['access_vector_path'])
scenarios = ['_bas_', '_cur_', '_fut_']
for lulc_val, scenario in enumerate(scenarios, start=1):
lulc_array = numpy.ones((100, 100), dtype=numpy.int8)
lulc_array[50:, :] = lulc_val
args['lulc' + scenario + 'path'] = os.path.join(
args['workspace_dir'], 'lc_samp' + scenario + 'b.tif')
make_raster_from_array(
lulc_array, args['lulc' + scenario + 'path'])
args['sensitivity_table_path'] = os.path.join(
args['workspace_dir'], 'sensitivity_samp.csv')
make_sensitivity_samp_csv(args['sensitivity_table_path'])
make_threats_raster(args['workspace_dir'])
args['threats_table_path'] = os.path.join(
args['workspace_dir'], 'threats_samp.csv')
# create the threat CSV table
with open(args['threats_table_path'], 'w') as open_table:
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
self.assertEqual(len(validate_result), 1)
self.assertEqual(validate_result[0][0], ['threats_table_path'])
self.assertTrue(utils.matches_format_string(
validate_result[0][1],
habitat_quality.MISSING_WEIGHT_MSG))
def test_habitat_quality_validation_bad_threat_path(self):
"""Habitat Quality: test validation for bad threat paths."""
from natcap.invest import habitat_quality
@ -1421,9 +1574,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -1471,9 +1624,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1524,9 +1677,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1577,9 +1730,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_cur.tif,threat_1_c.tif\n')
'40,0.7,threat_1,linear,,threat_1_cur.tif,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1627,9 +1780,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -1680,9 +1833,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -1746,9 +1899,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_c.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -1811,9 +1964,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_c.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(ValueError) as cm:
@ -1901,9 +2054,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args)
@ -1980,9 +2133,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args)
@ -2029,9 +2182,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,threat_1_cur.tif,threat_1_c.tif\n')
'40,0.7,threat_1,threat_1_cur.tif,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,threat_2_c.tif,threat_2_f.tif\n')
'70,1.0,threat_2,threat_2_c.tif,threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
expected = [(
@ -2080,9 +2233,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,threat_2_c.tif,'
'70,1.0,threat_2,exponential,threat_2_c.tif,'
'threat_2_f.tif\n')
validate_result = habitat_quality.validate(args, limit_to=None)
@ -2133,9 +2286,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif')
'70,1.0,threat_2,exponential,,threat_2_c.tif')
validate_result = habitat_quality.validate(args, limit_to=None)
expected = [(
@ -2146,7 +2299,7 @@ class HabitatQualityTests(unittest.TestCase):
self.assertEqual(validate_result, expected)
def test_habitat_quality_missing_lulc_val_in_sens_table(self):
"""Habitat Quality: test for empty value in LULC column of
"""Habitat Quality: test for empty value in lucode column of
sensitivity table. Expects TypeError"""
from natcap.invest import habitat_quality
@ -2165,9 +2318,9 @@ class HabitatQualityTests(unittest.TestCase):
args['sensitivity_table_path'] = os.path.join(
args['workspace_dir'], 'sensitivity_samp.csv')
with open(args['sensitivity_table_path'], 'w') as open_table:
open_table.write('LULC,NAME,HABITAT,threat_1,threat_2\n')
open_table.write('lucode,name,habitat,threat_1,threat_2\n')
open_table.write('1,"lulc 1",1,1,1\n')
open_table.write(',"lulc 2",0.5,0.5,1\n') # missing LULC value
open_table.write(',"lulc 2",0.5,0.5,1\n') # missing lucode value
open_table.write('3,"lulc 3",0,0.3,1\n')
make_threats_raster(
@ -2182,9 +2335,9 @@ class HabitatQualityTests(unittest.TestCase):
open_table.write(
'MAX_DIST,WEIGHT,THREAT,DECAY,BASE_PATH,CUR_PATH,FUT_PATH\n')
open_table.write(
'0.04,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
'40,0.7,threat_1,linear,,threat_1_c.tif,threat_1_f.tif\n')
open_table.write(
'0.07,1.0,threat_2,exponential,,threat_2_c.tif,'
'70,1.0,threat_2,exponential,,threat_2_c.tif,'
'threat_2_f.tif\n')
with self.assertRaises(TypeError):

28
tests/test_model_specs.py
View File

@ -50,7 +50,8 @@ class ValidateModelSpecs(unittest.TestCase):
def test_model_specs_are_valid(self):
"""MODEL_SPEC: test each spec meets the expected pattern."""
required_keys = {'model_name', 'pyname', 'userguide', 'args', 'outputs'}
required_keys = {
'model_id', 'model_name', 'pyname', 'userguide', 'args', 'outputs'}
optional_spatial_key = 'args_with_spatial_overlap'
for model_name, metadata in MODEL_METADATA.items():
# metadata is a collections.namedtuple, fields accessible by name
@ -229,7 +230,6 @@ class ValidateModelSpecs(unittest.TestCase):
raise AssertionError(f'{key} has key(s) {attrs} that are not '
'expected for its type')
def validate_args(self, arg, name, parent_type=None):
"""
Recursively validate nested args against the arg spec standard.
@ -505,29 +505,5 @@ class ValidateModelSpecs(unittest.TestCase):
f'{error}')
class SpecUtilsTests(unittest.TestCase):
"""Tests for natcap.invest.spec_utils."""
def test_format_unit(self):
"""spec_utils: test converting units to strings with format_unit."""
from natcap.invest import spec_utils
for unit_name, expected in [
('meter', 'm'),
('meter / second', 'm/s'),
('foot * mm', 'ft · mm'),
('t * hr * ha / ha / MJ / mm', 't · h · ha / (ha · MJ · mm)'),
('mm^3 / year', 'mm³/year')
]:
unit = spec_utils.u.Unit(unit_name)
actual = spec_utils.format_unit(unit)
self.assertEqual(expected, actual)
def test_format_unit_raises_error(self):
"""spec_utils: format_unit raises TypeError if not a pint.Unit."""
from natcap.invest import spec_utils
with self.assertRaises(TypeError):
spec_utils.format_unit({})
if __name__ == '__main__':
unittest.main()

180
tests/test_ndr.py
View File

@ -1,11 +1,11 @@
"""InVEST NDR model tests."""
import collections
import os
import shutil
import tempfile
import unittest
import numpy
import pandas
import pygeoprocessing
import shapely.geometry
from osgeo import gdal
@ -48,6 +48,7 @@ class NDRTests(unittest.TestCase):
'watersheds_path':
os.path.join(REGRESSION_DATA, 'input', 'watersheds.shp'),
'workspace_dir': workspace_dir,
'flow_dir_algorithm': 'MFD'
}
return args.copy()
@ -148,13 +149,13 @@ class NDRTests(unittest.TestCase):
if not feature:
raise AssertionError("No features were output.")
for field, value in [
('p_surface_load', 41.921),
('p_surface_export', 5.59887886),
('n_surface_load', 2978.520),
('n_subsurface_load', 28.614),
('n_surface_export', 289.0498),
('n_subsurface_load', 28.614094),
('n_total_export', 304.66061401)]:
('p_surface_load', 41.826904),
('p_surface_export', 5.566120),
('n_surface_load', 2977.551270),
('n_surface_export', 274.020844),
('n_subsurface_load', 28.558048),
('n_subsurface_export', 15.578484),
('n_total_export', 289.599314)]:
if not numpy.isclose(feature.GetField(field), value, atol=1e-2):
error_results[field] = (
'field', feature.GetField(field), value)
@ -195,11 +196,12 @@ class NDRTests(unittest.TestCase):
self.assertEqual(len(validation_messages), 1)
def test_base_regression(self):
"""NDR base regression test on sample data.
"""NDR base regression test on test data.
Execute NDR with sample data and checks that the output files are
generated and that the aggregate shapefile fields are the same as the
regression case.
Executes NDR with test data. Checks for accuracy of aggregate
values in summary vector, presence of drainage raster in
intermediate outputs, and accuracy of raster outputs (as
measured by the sum of their non-nodata pixel values).
"""
from natcap.invest.ndr import ndr
@ -213,6 +215,86 @@ class NDRTests(unittest.TestCase):
f.write(b'')
ndr.execute(args)
result_vector = ogr.Open(os.path.join(
args['workspace_dir'], 'watershed_results_ndr.gpkg'))
result_layer = result_vector.GetLayer()
result_feature = result_layer.GetFeature(1)
result_layer = None
result_vector = None
mismatch_list = []
# these values were generated by manual inspection of regression
# results
expected_watershed_totals = {
'p_surface_load': 41.826904,
'p_surface_export': 5.870544,
'n_surface_load': 2977.551270,
'n_surface_export': 274.020844,
'n_subsurface_load': 28.558048,
'n_subsurface_export': 15.578484,
'n_total_export': 289.599314
}
for field in expected_watershed_totals:
expected_value = expected_watershed_totals[field]
val = result_feature.GetField(field)
if not numpy.isclose(val, expected_value):
mismatch_list.append(
(field, 'expected: %f' % expected_value,
'actual: %f' % val))
result_feature = None
if mismatch_list:
raise AssertionError("results not expected: %s" % mismatch_list)
# We only need to test that the drainage mask exists. Functionality
# for that raster is tested in SDR.
self.assertTrue(
os.path.exists(
os.path.join(
args['workspace_dir'], 'intermediate_outputs',
'what_drains_to_stream.tif')))
# Check raster outputs to make sure values are in kg/ha/yr.
raster_info = pygeoprocessing.get_raster_info(args['dem_path'])
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
pixels_per_hectare = 10000 / pixel_area
for attr_name in ['p_surface_export',
'n_surface_export',
'n_subsurface_export',
'n_total_export']:
# Since pixel values are kg/(ha•yr), raster sum is (kg•px)/(ha•yr),
# equal to the watershed total (kg/yr) * (pixels_per_hectare px/ha).
expected_sum = (expected_watershed_totals[attr_name]
* pixels_per_hectare)
raster_name = attr_name + '.tif'
raster_path = os.path.join(args['workspace_dir'], raster_name)
nodata = pygeoprocessing.get_raster_info(raster_path)['nodata'][0]
raster_sum = 0.0
for _, block in pygeoprocessing.iterblocks((raster_path, 1)):
raster_sum += numpy.sum(
block[~pygeoprocessing.array_equals_nodata(
block, nodata)], dtype=numpy.float64)
numpy.testing.assert_allclose(raster_sum, expected_sum, rtol=1e-6)
def test_base_regression_d8(self):
"""NDR base regression test on sample data in D8 mode.
Execute NDR with sample data and checks that the output files are
generated and that the aggregate shapefile fields are the same as the
regression case.
"""
from natcap.invest.ndr import ndr
# use predefined directory so test can clean up files during teardown
args = NDRTests.generate_base_args(self.workspace_dir)
args['flow_dir_algorithm'] = 'D8'
# make an empty output shapefile on top of where the new output
# shapefile should reside to ensure the model overwrites it
with open(
os.path.join(self.workspace_dir, 'watershed_results_ndr.gpkg'),
'wb') as f:
f.write(b'')
ndr.execute(args)
result_vector = ogr.Open(os.path.join(
args['workspace_dir'], 'watershed_results_ndr.gpkg'))
result_layer = result_vector.GetLayer()
@ -223,13 +305,13 @@ class NDRTests(unittest.TestCase):
# these values were generated by manual inspection of regression
# results
for field, expected_value in [
('p_surface_load', 41.921860),
('p_surface_export', 5.899117),
('n_surface_load', 2978.519775),
('n_surface_export', 289.0498),
('n_subsurface_load', 28.614094),
('n_subsurface_export', 15.61077),
('n_total_export', 304.660614)]:
('p_surface_load', 41.826904),
('p_surface_export', 4.915544),
('n_surface_load', 2977.551914),
('n_surface_export', 320.082319),
('n_subsurface_load', 28.558048),
('n_subsurface_export', 12.609187),
('n_total_export', 330.293407)]:
val = result_feature.GetField(field)
if not numpy.isclose(val, expected_value):
mismatch_list.append(
@ -278,13 +360,13 @@ class NDRTests(unittest.TestCase):
# these values were generated by manual inspection of regression
# results
for field, expected_value in [
('p_surface_load', 41.921860),
('p_surface_export', 5.899117),
('n_surface_load', 2978.519775),
('n_surface_export', 289.0498),
('n_subsurface_load', 28.614094),
('n_subsurface_export', 15.61077),
('n_total_export', 304.660614)]:
('p_surface_load', 41.826904),
('p_surface_export', 5.870544),
('n_surface_load', 2977.551270),
('n_surface_export', 274.020844),
('n_subsurface_load', 28.558048),
('n_subsurface_export', 15.578484),
('n_total_export', 289.599314)]:
val = result_feature.GetField(field)
if not numpy.isclose(val, expected_value):
mismatch_list.append(
@ -362,6 +444,7 @@ class NDRTests(unittest.TestCase):
'k_param',
'watersheds_path',
'subsurface_eff_n',
'flow_dir_algorithm'
]
self.assertEqual(set(invalid_args), set(expected_missing_args))
@ -409,3 +492,48 @@ class NDRTests(unittest.TestCase):
numpy.testing.assert_array_equal(
expected_array,
pygeoprocessing.raster_to_numpy_array(target_raster_path))
def test_synthetic_runoff_proxy_av(self):
"""
Test RPI given user-entered or auto-calculated runoff proxy average.
Test that the runoff proxy index (RPI) is calculated correctly if
(1) the user specifies a runoff proxy average value,
(2) the user does not specify a value so the runoff proxy average
is auto-calculated.
"""
from natcap.invest.ndr import ndr
# make simple raster
runoff_proxy_path = os.path.join(self.workspace_dir, "ppt.tif")
runoff_proxy_array = numpy.array(
[[800, 799, 567, 234], [765, 867, 765, 654]], dtype=numpy.float32)
srs = osr.SpatialReference()
srs.ImportFromEPSG(26910)
projection_wkt = srs.ExportToWkt()
origin = (461251, 4923445)
pixel_size = (30, -30)
no_data = -1
pygeoprocessing.numpy_array_to_raster(
runoff_proxy_array, no_data, pixel_size, origin, projection_wkt,
runoff_proxy_path)
target_rpi_path = os.path.join(self.workspace_dir, "out_raster.tif")
# Calculate RPI with user-specified runoff proxy average
runoff_proxy_av = 2
ndr._normalize_raster((runoff_proxy_path, 1), target_rpi_path,
user_provided_mean=runoff_proxy_av)
actual_rpi = pygeoprocessing.raster_to_numpy_array(target_rpi_path)
expected_rpi = runoff_proxy_array/runoff_proxy_av
numpy.testing.assert_allclose(actual_rpi, expected_rpi)
# Now calculate RPI with auto-calculated RP average
ndr._normalize_raster((runoff_proxy_path, 1), target_rpi_path,
user_provided_mean=None)
actual_rpi = pygeoprocessing.raster_to_numpy_array(target_rpi_path)
expected_rpi = runoff_proxy_array/numpy.mean(runoff_proxy_array)
numpy.testing.assert_allclose(actual_rpi, expected_rpi)

1333
tests/test_recreation.py
View File

File diff suppressed because it is too large Load Diff

103
tests/test_sdr.py
View File

@ -71,6 +71,7 @@ class SDRTests(unittest.TestCase):
'watersheds_path': os.path.join(SAMPLE_DATA, 'watersheds.shp'),
'workspace_dir': workspace_dir,
'n_workers': -1,
'flow_dir_algorithm': 'MFD'
}
return args
@ -105,15 +106,6 @@ class SDRTests(unittest.TestCase):
validate_result, ['GDAL raster', 'GDAL vector']):
self.assertTrue(phrase in error_msg)
def test_sdr_validation_missing_key(self):
"""SDR test validation that's missing keys."""
from natcap.invest.sdr import sdr
# use predefined directory so test can clean up files during teardown
args = {}
validation_warnings = sdr.validate(args, limit_to=None)
self.assertEqual(len(validation_warnings[0][0]), 12)
def test_sdr_validation_key_no_value(self):
"""SDR test validation that's missing a value on a key."""
from natcap.invest.sdr import sdr
@ -128,7 +120,80 @@ class SDRTests(unittest.TestCase):
'expected a validation error but didn\'t get one')
def test_base_regression(self):
"""SDR base regression test on sample data.
"""SDR base regression test on test data.
Executes SDR with test data. Checks for accuracy of aggregate
values in summary vector, presence of drainage raster in
intermediate outputs, absence of negative (non-nodata) values
in sed_deposition raster, and accuracy of raster outputs (as
measured by the sum of their non-nodata pixel values).
"""
from natcap.invest.sdr import sdr
# use predefined directory so test can clean up files during teardown
args = SDRTests.generate_base_args(self.workspace_dir)
sdr.execute(args)
expected_watershed_totals = {
'usle_tot': 2.62457418442,
'sed_export': 0.09748090804,
'sed_dep': 1.71672844887,
'avoid_exp': 10199.46875,
'avoid_eros': 274444.75,
}
vector_path = os.path.join(
args['workspace_dir'], 'watershed_results_sdr.shp')
assert_expected_results_in_vector(expected_watershed_totals,
vector_path)
# We only need to test that the drainage mask exists. Functionality
# for that raster is tested elsewhere
self.assertTrue(
os.path.exists(
os.path.join(
args['workspace_dir'], 'intermediate_outputs',
'what_drains_to_stream.tif')))
# Check that sed_deposition does not have any negative, non-nodata
# values, even if they are very small.
sed_deposition_path = os.path.join(args['workspace_dir'],
'sed_deposition.tif')
sed_dep_nodata = pygeoprocessing.get_raster_info(
sed_deposition_path)['nodata'][0]
sed_dep_array = pygeoprocessing.raster_to_numpy_array(
sed_deposition_path)
negative_non_nodata_mask = (
(~numpy.isclose(sed_dep_array, sed_dep_nodata)) &
(sed_dep_array < 0))
self.assertEqual(
numpy.count_nonzero(sed_dep_array[negative_non_nodata_mask]), 0)
# Check raster outputs to make sure values are in Mg/ha/yr.
raster_info = pygeoprocessing.get_raster_info(args['dem_path'])
pixel_area = abs(numpy.prod(raster_info['pixel_size']))
pixels_per_hectare = 10000 / pixel_area
for (raster_name,
attr_name) in [('usle.tif', 'usle_tot'),
('sed_export.tif', 'sed_export'),
('sed_deposition.tif', 'sed_dep'),
('avoided_export.tif', 'avoid_exp'),
('avoided_erosion.tif', 'avoid_eros')]:
# Since pixel values are Mg/(ha•yr), raster sum is (Mg•px)/(ha•yr),
# equal to the watershed total (Mg/yr) * (pixels_per_hectare px/ha).
expected_sum = (expected_watershed_totals[attr_name]
* pixels_per_hectare)
raster_path = os.path.join(args['workspace_dir'], raster_name)
nodata = pygeoprocessing.get_raster_info(raster_path)['nodata'][0]
raster_sum = 0.0
for _, block in pygeoprocessing.iterblocks((raster_path, 1)):
raster_sum += numpy.sum(
block[~pygeoprocessing.array_equals_nodata(
block, nodata)], dtype=numpy.float64)
numpy.testing.assert_allclose(raster_sum, expected_sum)
def test_base_regression_d8(self):
"""SDR base regression test on sample data in D8 mode.
Execute SDR with sample data and checks that the output files are
generated and that the aggregate shapefile fields are the same as the
@ -138,15 +203,17 @@ class SDRTests(unittest.TestCase):
# use predefined directory so test can clean up files during teardown
args = SDRTests.generate_base_args(self.workspace_dir)
args['flow_dir_algorithm'] = 'D8'
args['threshold_flow_accumulation'] = 100
# make args explicit that this is a base run of SWY
sdr.execute(args)
expected_results = {
'usle_tot': 2.62457418442,
'sed_export': 0.09748090804,
'sed_dep': 1.71672844887,
'avoid_exp': 10199.46875,
'avoid_eros': 274444.75,
'usle_tot': 2.520746,
'sed_export': 0.187428,
'sed_dep': 2.300645,
'avoid_exp': 19283.767578,
'avoid_eros': 263415,
}
vector_path = os.path.join(
@ -288,8 +355,8 @@ class SDRTests(unittest.TestCase):
with self.assertRaises(ValueError) as context:
sdr.execute(args)
self.assertIn(
f'A value in the biophysical table is not a number '
f'within range 0..1.', str(context.exception))
'A value in the biophysical table is not a number '
'within range 0..1.', str(context.exception))
def test_base_usle_p_nan(self):
"""SDR test expected exception for USLE_P not a number."""
@ -304,7 +371,7 @@ class SDRTests(unittest.TestCase):
with self.assertRaises(ValueError) as context:
sdr.execute(args)
self.assertIn(
f'could not be interpreted as ratios', str(context.exception))
'could not be interpreted as ratios', str(context.exception))
def test_lucode_not_a_number(self):
"""SDR test expected exception for invalid data in lucode column."""

72
tests/test_seasonal_water_yield_regression.py
View File

@ -424,6 +424,7 @@ class SeasonalWaterYieldUnusualDataTests(unittest.TestCase):
'user_defined_climate_zones': False,
'user_defined_local_recharge': False,
'monthly_alpha': False,
'flow_dir_algorithm': 'MFD'
}
watershed_shp_path = os.path.join(args['workspace_dir'],
@ -485,6 +486,7 @@ class SeasonalWaterYieldUnusualDataTests(unittest.TestCase):
'user_defined_climate_zones': False,
'user_defined_local_recharge': False,
'monthly_alpha': False,
'flow_dir_algorithm': 'MFD'
}
watershed_shp_path = os.path.join(args['workspace_dir'],
@ -585,6 +587,7 @@ class SeasonalWaterYieldUnusualDataTests(unittest.TestCase):
'user_defined_climate_zones': False,
'user_defined_local_recharge': False,
'monthly_alpha': False,
'flow_dir_algorithm': 'MFD'
}
biophysical_csv_path = os.path.join(args['workspace_dir'],
@ -644,6 +647,7 @@ class SeasonalWaterYieldRegressionTests(unittest.TestCase):
'results_suffix': '',
'threshold_flow_accumulation': '50',
'workspace_dir': workspace_dir,
'flow_dir_algorithm': 'MFD'
}
watershed_shp_path = os.path.join(workspace_dir, 'watershed.shp')
@ -730,6 +734,58 @@ class SeasonalWaterYieldRegressionTests(unittest.TestCase):
os.path.join(args['workspace_dir'], 'aggregated_results_swy.shp'),
agg_results_csv_path)
def test_base_regression_d8(self):
"""SWY base regression test on sample data in D8 mode.
Executes SWY in default mode and checks that the output files are
generated and that the aggregate shapefile fields are the same as the
regression case.
"""
from natcap.invest.seasonal_water_yield import seasonal_water_yield
# use predefined directory so test can clean up files during teardown
args = SeasonalWaterYieldRegressionTests.generate_base_args(
self.workspace_dir)
# Ensure the model can pass when a nodata value is not defined.
size = 100
lulc_array = numpy.zeros((size, size), dtype=numpy.int8)
lulc_array[size // 2:, :] = 1
driver = gdal.GetDriverByName('GTiff')
new_raster = driver.Create(
args['lulc_raster_path'], lulc_array.shape[0],
lulc_array.shape[1], 1, gdal.GDT_Byte)
band = new_raster.GetRasterBand(1)
band.WriteArray(lulc_array)
geotransform = [1180000, 1, 0, 690000, 0, -1]
new_raster.SetGeoTransform(geotransform)
band = None
new_raster = None
driver = None
# make args explicit that this is a base run of SWY
args['user_defined_climate_zones'] = False
args['user_defined_local_recharge'] = False
args['monthly_alpha'] = False
args['results_suffix'] = ''
args['flow_dir_algorithm'] = 'D8'
seasonal_water_yield.execute(args)
result_vector = ogr.Open(os.path.join(
args['workspace_dir'], 'aggregated_results_swy.shp'))
result_layer = result_vector.GetLayer()
result_feature = result_layer.GetFeature(0)
mismatch_list = []
for field, expected_value in [('vri_sum', 1), ('qb', 52.9128)]:
val = result_feature.GetField(field)
if not numpy.isclose(val, expected_value):
mismatch_list.append(
(field, f'expected: {expected_value}', f'actual: {val}'))
if mismatch_list:
raise RuntimeError(f'results not expected: {mismatch_list}')
def test_base_regression_nodata_inf(self):
"""SWY base regression test on sample data with really small nodata.
@ -1276,7 +1332,8 @@ class SeasonalWaterYieldRegressionTests(unittest.TestCase):
[kc_path for i in range(12)], alpha_month_map, beta,
gamma, stream_path, target_li_path, target_li_avail_path,
target_l_sum_avail_path, target_aet_path,
os.path.join(self.workspace_dir, 'target_precip_path.tif'))
os.path.join(self.workspace_dir, 'target_precip_path.tif'),
algorithm='MFD')
actual_li = pygeoprocessing.raster_to_numpy_array(target_li_path)
actual_li_avail = pygeoprocessing.raster_to_numpy_array(target_li_avail_path)
@ -1284,7 +1341,7 @@ class SeasonalWaterYieldRegressionTests(unittest.TestCase):
actual_aet = pygeoprocessing.raster_to_numpy_array(target_aet_path)
# note: obtained these arrays by running `calculate_local_recharge`
expected_li = numpy.array([[60., -72., 73.915215],
expected_li = numpy.array([[60., -72., 73.91521],
[0, 76.68, 828.]])
expected_li_avail = numpy.array([[30., -72., 36.957607],
[0, 38.34, 414.]])
@ -1352,16 +1409,16 @@ class SeasonalWaterYieldRegressionTests(unittest.TestCase):
seasonal_water_yield_core.route_baseflow_sum(flow_dir_mfd_path, l_path,
l_avail_path, l_sum_path,
stream_path, target_b_path,
target_b_sum_path)
target_b_sum_path, 'MFD')
actual_b = pygeoprocessing.raster_to_numpy_array(target_b_path)
actual_b_sum = pygeoprocessing.raster_to_numpy_array(target_b_sum_path)
# note: obtained these arrays by running `route_baseflow_sum`
expected_b = numpy.array([[10.5, 0.9999998, 0],
[0.1422222, 2.2666667, 0]])
expected_b_sum = numpy.array([[16.916666, 1.8666663, 0],
[0.1422222, 2.5333333, 0]])
expected_b = numpy.array([[10.5, 1, 0],
[0.14222223, 2.2666667, 0]])
expected_b_sum = numpy.array([[16.916666, 1.8666667, 0],
[0.14222223, 2.5333333, 0]])
numpy.testing.assert_allclose(actual_b, expected_b, equal_nan=True,
err_msg="Baseflow raster values do not match.")
@ -1432,6 +1489,7 @@ class SWYValidationTests(unittest.TestCase):
'precip_dir',
'threshold_flow_accumulation',
'user_defined_local_recharge',
'flow_dir_algorithm'
]
def tearDown(self):

146
tests/test_spec_utils.py
View File

@ -1,12 +1,44 @@
import os
import shutil
import tempfile
import types
import unittest
import geometamaker
from natcap.invest import spec_utils
from natcap.invest.unit_registry import u
from osgeo import gdal
from osgeo import ogr
gdal.UseExceptions()
class TestSpecUtils(unittest.TestCase):
class SpecUtilsUnitTests(unittest.TestCase):
"""Unit tests for natcap.invest.spec_utils."""
def test_format_unit(self):
"""spec_utils: test converting units to strings with format_unit."""
from natcap.invest import spec_utils
for unit_name, expected in [
('meter', 'm'),
('meter / second', 'm/s'),
('foot * mm', 'ft · mm'),
('t * hr * ha / ha / MJ / mm', 't · h · ha / (ha · MJ · mm)'),
('mm^3 / year', 'mm³/year')
]:
unit = spec_utils.u.Unit(unit_name)
actual = spec_utils.format_unit(unit)
self.assertEqual(expected, actual)
def test_format_unit_raises_error(self):
"""spec_utils: format_unit raises TypeError if not a pint.Unit."""
from natcap.invest import spec_utils
with self.assertRaises(TypeError):
spec_utils.format_unit({})
class TestDescribeArgFromSpec(unittest.TestCase):
"""Test building RST for various invest args specifications."""
def test_number_spec(self):
spec = {
@ -256,3 +288,115 @@ class TestSpecUtils(unittest.TestCase):
carbon.MODEL_SPEC['args']['carbon_pools_path']['columns']['lucode']['about']
)
self.assertEqual(repr(out), repr(expected_rst))
def _generate_files_from_spec(output_spec, workspace):
"""A utility function to support the metadata test."""
for filename, spec_data in output_spec.items():
if 'type' in spec_data and spec_data['type'] == 'directory':
os.mkdir(os.path.join(workspace, filename))
_generate_files_from_spec(
spec_data['contents'], os.path.join(workspace, filename))
else:
filepath = os.path.join(workspace, filename)
if 'bands' in spec_data:
driver = gdal.GetDriverByName('GTIFF')
n_bands = len(spec_data['bands'])
raster = driver.Create(
filepath, 2, 2, n_bands, gdal.GDT_Byte)
for i in range(n_bands):
band = raster.GetRasterBand(i + 1)
band.SetNoDataValue(2)
elif 'fields' in spec_data:
if 'geometries' in spec_data:
driver = gdal.GetDriverByName('GPKG')
target_vector = driver.CreateDataSource(filepath)
layer_name = os.path.basename(os.path.splitext(filepath)[0])
target_layer = target_vector.CreateLayer(
layer_name, geom_type=ogr.wkbPolygon)
for field_name, field_data in spec_data['fields'].items():
target_layer.CreateField(ogr.FieldDefn(field_name, ogr.OFTInteger))
else:
# Write a CSV if it has fields but no geometry
with open(filepath, 'w') as file:
file.write(
f"{','.join([field for field in spec_data['fields']])}")
else:
# Such as taskgraph.db, just create the file.
with open(filepath, 'w') as file:
pass
class TestMetadataFromSpec(unittest.TestCase):
"""Tests for metadata-generation functions."""
def setUp(self):
"""Override setUp function to create temp workspace directory."""
self.workspace_dir = tempfile.mkdtemp()
def tearDown(self):
"""Override tearDown function to remove temporary directory."""
shutil.rmtree(self.workspace_dir)
def test_write_metadata(self):
"""Test writing metadata for an invest output workspace."""
# An example invest output spec
output_spec = {
'output': {
"type": "directory",
"contents": {
"urban_nature_supply_percapita.tif": {
"about": (
"The calculated supply per capita of urban nature."),
"bands": {1: {
"type": "number",
"units": u.m**2,
}}},
"admin_boundaries.gpkg": {
"about": (
"A copy of the user's administrative boundaries "
"vector with a single layer."),
"geometries": spec_utils.POLYGONS,
"fields": {
"SUP_DEMadm_cap": {
"type": "number",
"units": u.m**2/u.person,
"about": (
"The average urban nature supply/demand ")
}
}
}
},
},
'intermediate': {
'type': 'directory',
'contents': {
'taskgraph_cache': spec_utils.TASKGRAPH_DIR,
}
}
}
# Generate an output workspace with real files, without
# running an invest model.
_generate_files_from_spec(output_spec, self.workspace_dir)
model_module = types.SimpleNamespace(
__name__='urban_nature_access',
execute=lambda: None,
MODEL_SPEC={
'model_id': 'urban_nature_access',
'outputs': output_spec})
args_dict = {'workspace_dir': self.workspace_dir}
spec_utils.generate_metadata(model_module, args_dict)
files, messages = geometamaker.validate_dir(
self.workspace_dir, recursive=True)
self.assertEqual(len(files), 2)
self.assertFalse(any(messages))
resource = geometamaker.describe(
os.path.join(args_dict['workspace_dir'], 'output',
'urban_nature_supply_percapita.tif'))
self.assertCountEqual(resource.get_keywords(),
[model_module.MODEL_SPEC['model_id'], 'InVEST'])

8
tests/test_translation.py
View File

@ -24,7 +24,7 @@ TEST_MESSAGES = {
"InVEST Carbon Model": "ιиνєѕт ςαявσи мσ∂єℓ",
"Available models:": "αναιℓαвℓє мσ∂єℓѕ:",
"Carbon Storage and Sequestration": "ςαявσи ѕтσяαgє αи∂ ѕєףυєѕтяαтισи",
"current LULC": "ςυяяєит ℓυℓς",
"baseline LULC": "вαѕєℓιиє ℓυℓς",
missing_key_msg: "кєу ιѕ мιѕѕιиg fяσм тнє αяgѕ ∂ιςт",
not_a_number_msg: 'ναℓυє "{value}" ςσυℓ∂ иσт вє ιитєяρяєтє∂ αѕ α иυмвєя'
}
@ -94,7 +94,7 @@ class TranslationTests(unittest.TestCase):
with self.assertRaises(SystemExit):
cli.main(['--language', TEST_LANG, 'getspec', 'carbon'])
result = out.getvalue()
self.assertIn(TEST_MESSAGES['current LULC'], result)
self.assertIn(TEST_MESSAGES['baseline LULC'], result)
def test_invest_validate(self):
"""Translation: test that CLI validate output is translated."""
@ -134,8 +134,8 @@ class TranslationTests(unittest.TestCase):
'api/getspec', json='carbon', query_string={'language': TEST_LANG})
spec = json.loads(response.get_data(as_text=True))
self.assertEqual(
spec['args']['lulc_cur_path']['name'],
TEST_MESSAGES['current LULC'])
spec['args']['lulc_bas_path']['name'],
TEST_MESSAGES['baseline LULC'])
def test_server_get_invest_validate(self):
"""Translation: test that /validate endpoint is translated."""

24
tests/test_ucm.py
View File

@ -119,9 +119,9 @@ class UCMTests(unittest.TestCase):
results_feature = results_layer.GetFeature(1)
expected_results = {
'avg_cc': 0.222150472947109,
'avg_tmp_v': 37.306549,
'avg_tmp_an': 2.306549,
'avg_cc': 0.221991,
'avg_tmp_v': 37.303789,
'avg_tmp_an': 2.303789,
'avd_eng_cn': 3602851.784639,
'avg_wbgt_v': 32.585935,
'avg_ltls_v': 75.000000000000000,
@ -141,7 +141,7 @@ class UCMTests(unittest.TestCase):
# Assert that the decimal value of the energy savings value is what we
# expect.
expected_energy_sav = 3647696.209368
expected_energy_sav = 3641030.461044
energy_sav = 0.0
n_nonetype = 0
@ -162,7 +162,7 @@ class UCMTests(unittest.TestCase):
# Expected energy savings is an accumulated value and may differ
# past about 4 decimal places.
numpy.testing.assert_allclose(energy_sav, expected_energy_sav, rtol=1e-4)
self.assertEqual(n_nonetype, 121)
self.assertEqual(n_nonetype, 136)
finally:
buildings_layer = None
buildings_vector = None
@ -201,7 +201,7 @@ class UCMTests(unittest.TestCase):
# and may differ past about 4 decimal places.
numpy.testing.assert_allclose(energy_sav, expected_energy_sav,
rtol=1e-4)
self.assertEqual(n_nonetype, 121)
self.assertEqual(n_nonetype, 136)
finally:
buildings_layer = None
buildings_vector = None
@ -249,12 +249,12 @@ class UCMTests(unittest.TestCase):
results_feature = results_layer.GetFeature(1)
expected_results = {
'avg_cc': 0.428302583240327,
'avg_tmp_v': 36.60869797039769,
'avg_tmp_an': 1.608697970397692,
'avd_eng_cn': 7239992.744486,
'avg_wbgt_v': 31.91108630952381,
'avg_ltls_v': 28.73463901689708,
'avg_cc': 0.422250,
'avg_tmp_v': 36.621779,
'avg_tmp_an': 1.621779,
'avd_eng_cn': 7148968.928616,
'avg_wbgt_v': 31.92365,
'avg_ltls_v': 29.380548,
'avg_hvls_v': 75.000000000000000,
}
try:

56
tests/test_ui_server.py
View File

@ -34,8 +34,8 @@ class EndpointFunctionTests(unittest.TestCase):
# an empty path
response = test_client.post(
f'{ROUTE_PREFIX}/colnames', json={'vector_path': ''})
colnames = json.loads(response.get_data(as_text=True))
self.assertEqual(response.status_code, 422)
colnames = json.loads(response.get_data(as_text=True))
self.assertEqual(colnames, [])
# a vector with one column
path = os.path.join(
@ -43,20 +43,22 @@ class EndpointFunctionTests(unittest.TestCase):
'watersheds.shp')
response = test_client.post(
f'{ROUTE_PREFIX}/colnames', json={'vector_path': path})
self.assertEqual(response.status_code, 200)
colnames = json.loads(response.get_data(as_text=True))
self.assertEqual(colnames, ['ws_id'])
# a non-vector file
path = os.path.join(TEST_DATA_PATH, 'ndr', 'input', 'dem.tif')
response = test_client.post(
f'{ROUTE_PREFIX}/colnames', json={'vector_path': path})
colnames = json.loads(response.get_data(as_text=True))
self.assertEqual(response.status_code, 422)
colnames = json.loads(response.get_data(as_text=True))
self.assertEqual(colnames, [])
def test_get_invest_models(self):
"""UI server: get_invest_models endpoint."""
test_client = ui_server.app.test_client()
response = test_client.get(f'{ROUTE_PREFIX}/models')
self.assertEqual(response.status_code, 200)
models_dict = json.loads(response.get_data(as_text=True))
for model in models_dict.values():
self.assertEqual(set(model), {'model_name', 'aliases'})
@ -65,10 +67,11 @@ class EndpointFunctionTests(unittest.TestCase):
"""UI server: get_invest_spec endpoint."""
test_client = ui_server.app.test_client()
response = test_client.post(f'{ROUTE_PREFIX}/getspec', json='sdr')
self.assertEqual(response.status_code, 200)
spec = json.loads(response.get_data(as_text=True))
self.assertEqual(
set(spec),
{'model_name', 'pyname', 'userguide',
{'model_id', 'model_name', 'pyname', 'userguide',
'args_with_spatial_overlap', 'args', 'outputs'})
def test_get_invest_validate(self):
@ -83,6 +86,7 @@ class EndpointFunctionTests(unittest.TestCase):
'args': json.dumps(args)
}
response = test_client.post(f'{ROUTE_PREFIX}/validate', json=payload)
self.assertEqual(response.status_code, 200)
results = json.loads(response.get_data(as_text=True))
expected = carbon.validate(args)
# These differ only because a tuple was transformed to a list during
@ -92,7 +96,6 @@ class EndpointFunctionTests(unittest.TestCase):
def test_post_datastack_file(self):
"""UI server: post_datastack_file endpoint."""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
expected_datastack = {
'args': {
'workspace_dir': 'foo'
@ -105,6 +108,7 @@ class EndpointFunctionTests(unittest.TestCase):
file.write(json.dumps(expected_datastack))
response = test_client.post(
f'{ROUTE_PREFIX}/post_datastack_file', json={'filepath': filepath})
self.assertEqual(response.status_code, 200)
response_data = json.loads(response.get_data(as_text=True))
self.assertEqual(
set(response_data),
@ -114,7 +118,6 @@ class EndpointFunctionTests(unittest.TestCase):
def test_write_parameter_set_file(self):
"""UI server: write_parameter_set_file endpoint."""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
filepath = os.path.join(self.workspace_dir, 'datastack.json')
payload = {
'filepath': filepath,
@ -126,6 +129,7 @@ class EndpointFunctionTests(unittest.TestCase):
}
response = test_client.post(
f'{ROUTE_PREFIX}/write_parameter_set_file', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json,
{'message': 'Parameter set saved', 'error': False})
@ -140,7 +144,6 @@ class EndpointFunctionTests(unittest.TestCase):
should catch a ValueError and return an error message.
"""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
filepath = os.path.join(self.workspace_dir, 'datastack.json')
payload = {
'filepath': filepath,
@ -155,6 +158,7 @@ class EndpointFunctionTests(unittest.TestCase):
side_effect=ValueError(error_message)):
response = test_client.post(
f'{ROUTE_PREFIX}/write_parameter_set_file', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json,
{'message': error_message, 'error': True})
@ -162,7 +166,6 @@ class EndpointFunctionTests(unittest.TestCase):
def test_save_to_python(self):
"""UI server: save_to_python endpoint."""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
filepath = os.path.join(self.workspace_dir, 'script.py')
payload = {
'filepath': filepath,
@ -171,14 +174,14 @@ class EndpointFunctionTests(unittest.TestCase):
'workspace_dir': 'foo'
}),
}
_ = test_client.post(f'{ROUTE_PREFIX}/save_to_python', json=payload)
response = test_client.post(f'{ROUTE_PREFIX}/save_to_python', json=payload)
self.assertEqual(response.status_code, 200)
# test_cli.py asserts the actual contents of the file
self.assertTrue(os.path.exists(filepath))
def test_build_datastack_archive(self):
"""UI server: build_datastack_archive endpoint."""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
target_filepath = os.path.join(self.workspace_dir, 'data.tgz')
data_path = os.path.join(self.workspace_dir, 'data.csv')
with open(data_path, 'w') as file:
@ -194,6 +197,7 @@ class EndpointFunctionTests(unittest.TestCase):
}
response = test_client.post(
f'{ROUTE_PREFIX}/build_datastack_archive', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json,
{'message': 'Datastack archive created', 'error': False})
@ -205,7 +209,6 @@ class EndpointFunctionTests(unittest.TestCase):
should catch a ValueError and return an error message.
"""
test_client = ui_server.app.test_client()
self.workspace_dir = tempfile.mkdtemp()
target_filepath = os.path.join(self.workspace_dir, 'data.tgz')
data_path = os.path.join(self.workspace_dir, 'data.csv')
with open(data_path, 'w') as file:
@ -224,6 +227,7 @@ class EndpointFunctionTests(unittest.TestCase):
side_effect=ValueError(error_message)):
response = test_client.post(
f'{ROUTE_PREFIX}/build_datastack_archive', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json,
{'message': error_message, 'error': True})
@ -247,6 +251,7 @@ class EndpointFunctionTests(unittest.TestCase):
}
response = test_client.post(
f'{ROUTE_PREFIX}/log_model_start', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_data(as_text=True), 'OK')
mock_get.assert_called_once()
mock_post.assert_called_once()
@ -276,8 +281,39 @@ class EndpointFunctionTests(unittest.TestCase):
}
response = test_client.post(
f'{ROUTE_PREFIX}/log_model_exit', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.get_data(as_text=True), 'OK')
mock_get.assert_called_once()
mock_post.assert_called_once()
self.assertEqual(mock_post.call_args.args[0], mock_url)
self.assertEqual(mock_post.call_args.kwargs['data'], payload)
@patch('natcap.invest.ui_server.geometamaker.config.platformdirs.user_config_dir')
def test_get_geometamaker_profile(self, mock_user_config_dir):
"""UI server: get_geometamaker_profile endpoint."""
test_client = ui_server.app.test_client()
response = test_client.get(f'{ROUTE_PREFIX}/get_geometamaker_profile')
self.assertEqual(response.status_code, 200)
profile_dict = json.loads(response.get_data(as_text=True))
self.assertIn('contact', profile_dict)
self.assertIn('license', profile_dict)
@patch('natcap.invest.ui_server.geometamaker.config.platformdirs.user_config_dir')
def test_set_geometamaker_profile(self, mock_user_config_dir):
"""UI server: set_geometamaker_profile endpoint."""
mock_user_config_dir.return_value = self.workspace_dir
test_client = ui_server.app.test_client()
payload = {
'contact': {
'individual_name': 'Foo'
},
'license': {
'title': 'Bar'
},
}
response = test_client.post(
f'{ROUTE_PREFIX}/set_geometamaker_profile', json=payload)
self.assertEqual(response.status_code, 200)
self.assertEqual(
response.json,
{'message': 'Metadata profile saved', 'error': False})

292
workbench/changelog.html
View File

@ -4,7 +4,7 @@
<meta charset="utf-8" />
</head>
<body>
<!-- Changes should be grouped for readability.
<!-- Changes should be grouped under headings for readability.
InVEST model names:
- Annual Water Yield
- Carbon Storage and Sequestration
@ -13,7 +13,7 @@ InVEST model names:
- Crop Pollination
- Crop Production
- DelineateIt
- Forest Carbon Edge Effects
- Forest Carbon Edge Effect
- Globio
- Habitat Quality
- HRA
@ -33,11 +33,173 @@ InVEST model names:
Workbench fixes/enhancements:
- Workbench
Everything else:
- General -->
<!-- :changelog: -->
- General
Updates worth drawing extra attention to (minor/major releases only):
- Highlights
Each release section has a heading underlined with ``---``.
Within a release section, underline group headings with ``===``.
For example:
Unreleased Changes
------------------
General
=======
* Updated something.
Wind Energy
===========
* Fixed something.
The order of groups should be as follows:
1. Highlights
2. General
3. Workbench
4. InVEST model A
5. ...
6. InVEST model Z (model names should be sorted A-Z) -->
<!-- Unreleased Changes
------------------ -->
<section id="section-1">
<h1>3.15.0 (2025-04-03)</h1>
<section id="highlights">
<h2>Highlights</h2>
<ul>
<li>Multiple models now use <strong>per-hectare</strong> units in their raster outputs. Prior to this update, these rasters reported <strong>per-pixel</strong> values. This change affects the following models:
<ul>
<li>Carbon</li>
<li>Crop Production</li>
<li>Forest Carbon Edge Effect</li>
<li>NDR</li>
<li>SDR</li>
</ul>
</li>
<li>NDR, SDR, and Seasonal Water Yield now support the D8 routing algorithm in addition to MFD.</li>
<li>Visitation: Recreation and Tourism model now includes Twitter data.</li>
<li>InVEST model outputs now include metadata. Open the '.yml' files in a text editor to read and add to the metadata.</li>
</ul>
</section>
<section id="general">
<h2>General</h2>
<ul>
<li>Fixed an issue where a user's PROJ_DATA environment variable could trigger a RuntimeError about a missing proj.db file. <a href="https://github.com/natcap/invest/issues/1742">https://github.com/natcap/invest/issues/1742</a></li>
<li>Now testing and building against Python 3.13. No longer testing and building with Python 3.8, which reached EOL. <a href="https://github.com/natcap/invest/issues/1755">https://github.com/natcap/invest/issues/1755</a></li>
<li>All InVEST model output data now include metadata sidecar files. These are '.yml' files with the same basename as the dataset they describe. <a href="https://github.com/natcap/invest/issues/1662">https://github.com/natcap/invest/issues/1662</a></li>
<li>InVEST's Windows binaries are now distributed once again with a valid signature, signed by Stanford University. <a href="https://github.com/natcap/invest/issues/1580">https://github.com/natcap/invest/issues/1580</a></li>
<li>InVEST's macOS disk image is now distributed once again with a valid signature, signed by Stanford University. <a href="https://github.com/natcap/invest/issues/1784">https://github.com/natcap/invest/issues/1784</a></li>
<li>The natcap.invest python package now officially supports linux. manylinux wheels will be available on PyPI. (<a href="https://github.com/natcap/invest/issues/1730">#1730</a>)</li>
<li>Removed the warning about <code>gdal.UseExceptions()</code>. Python API users should still call <code>gdal.UseExceptions()</code>, but no longer need to do so before importing <code>natcap.invest</code>. <a href="https://github.com/natcap/invest/issues/1702">https://github.com/natcap/invest/issues/1702</a></li>
</ul>
</section>
<section id="workbench">
<h2>Workbench</h2>
<ul>
<li>Auto-scrolling of log output is halted on user-initiated scrolling, enabling easier inspection of log output while a model is running (<a href="https://github.com/natcap/invest/issues/1533">InVEST #1533</a>).</li>
<li>Fixed a bug where toggle inputs would fail to respond if multiple tabs of the same model were open. <a href="https://github.com/natcap/invest/issues/1842">https://github.com/natcap/invest/issues/1842</a></li>
</ul>
</section>
<section id="annual-water-yield">
<h2>Annual Water Yield</h2>
<ul>
<li>Fixed an issue where the model would crash if the valuation table was provided, but the demand table was not. Validation will now warn about this, and the <code>MODEL_SPEC</code> has been improved to reflect that this table is now required when doing valuation. <a href="https://github.com/natcap/invest/issues/1769">https://github.com/natcap/invest/issues/1769</a></li>
</ul>
</section>
<section id="carbon">
<h2>Carbon</h2>
<ul>
<li>Updated styling of the HTML report generated by the carbon model, for visual consistency with the Workbench (<a href="https://github.com/natcap/invest/issues/1732">InVEST #1732</a>).</li>
<li>Raster outputs that previously contained per-pixel values (e.g., t/pixel) now contain per-hectare values (e.g., t/ha). (<a href="https://github.com/natcap/invest/issues/1270">InVEST #1270</a>).</li>
<li>Removed the REDD scenario and updated the naming of the Current and Future scenarios to Baseline and Alternate, respectively, to better indicate that users are not limited to comparing present and future. (<a href="https://github.com/natcap/invest/issues/1758">InVEST #1758</a>).</li>
<li>Changed output filename prefixes from <code>tot_c</code> to <code>c_storage</code> and <code>delta</code> to <code>c_change</code>. (<a href="https://github.com/natcap/invest/issues/1825">InVEST #1825</a>).</li>
<li>Fixed bug where discount rate and annual price change were incorrectly treated as ratios instead of percentages. (<a href="https://github.com/natcap/invest/issues/1827">InVEST #1827</a>).</li>
</ul>
</section>
<section id="coastal-blue-carbon">
<h2>Coastal Blue Carbon</h2>
<ul>
<li>The <code>code</code> column in the model's biophysical table input, as well as the <code>code</code> column in the preprocessor's LULC lookup table input and <code>carbon_pool_transient_template</code> output, have been renamed <code>lucode</code>, for consistency with other InVEST models (<a href="https://github.com/natcap/invest/issues/1249">InVEST #1249</a>).</li>
</ul>
</section>
<section id="crop-production">
<h2>Crop Production</h2>
<ul>
<li>Raster outputs that previously contained per-pixel values (e.g., t/pixel) now contain per-hectare values (e.g., t/ha). This change affects both the Percentile and Regression models (<a href="https://github.com/natcap/invest/issues/1270">InVEST #1270</a>).</li>
</ul>
</section>
<section id="forest-carbon-edge-effect">
<h2>Forest Carbon Edge Effect</h2>
<ul>
<li>Raster outputs that previously contained per-pixel values (e.g., t/pixel) now contain per-hectare values (e.g., t/ha). (<a href="https://github.com/natcap/invest/issues/1270">InVEST #1270</a>).</li>
</ul>
</section>
<section id="habitat-quality">
<h2>Habitat Quality</h2>
<ul>
<li>The <code>lulc</code> column in the sensitivity table input, and the <code>lulc_code</code> column in the rarity table outputs, have been renamed <code>lucode</code>, for consistency with other InVEST models (<a href="https://github.com/natcap/invest/issues/1249">InVEST #1249</a>).</li>
<li>The model now expects the maximum threat distance (<code>max_dist</code> in the threats table) to be specified in <code>m</code> instead of <code>km</code> (<a href="https://github.com/natcap/invest/issues/1252">InVEST #1252</a>).</li>
<li>Adjusted total habitat degradation calculation to calculate degradation for each threat and create intermediate degradation rasters. Total degradation is now calculated using these individual threat degradation rasters. <a href="https://github.com/natcap/invest/issues/1100">https://github.com/natcap/invest/issues/1100</a></li>
</ul>
</section>
<section id="ndr">
<h2>NDR</h2>
<ul>
<li>Align rasters to the grid of the DEM raster (<a href="https://github.com/natcap/invest/issues/1488">#1488</a>).</li>
<li>Raster outputs that previously contained per-pixel values (e.g., kg/pixel) now contain per-hectare values (e.g., kg/ha). (<a href="https://github.com/natcap/invest/issues/1270">InVEST #1270</a>).</li>
<li>Made the runoff proxy index calculation more robust by allowing users to specify the average runoff proxy, preventing normalization issues across different climate scenarios and watershed selections. <a href="https://github.com/natcap/invest/issues/1741">https://github.com/natcap/invest/issues/1741</a></li>
<li>D8 routing is now supported in addition to MFD (<a href="https://github.com/natcap/invest/issues/1440">#1440</a>).</li>
</ul>
</section>
<section id="scenario-generator">
<h2>Scenario Generator</h2>
<ul>
<li>Updated the output CSV columns: Renamed <cite>lucode</cite> column <cite>original lucode</cite> to clarify that it contains the original, to-be-converted, value(s). Added <cite>replacement lucode</cite> column, containing the LULC code to which habitat was converted during the model run. <a href="https://github.com/natcap/invest/issues/1295">https://github.com/natcap/invest/issues/1295</a></li>
</ul>
</section>
<section id="scenic-quality">
<h2>Scenic Quality</h2>
<ul>
<li>Fixed a bug where the visibility raster could be incorrectly set to 1 ('visible') if the DEM value was within floating point imprecision of the DEM nodata value (<a href="https://github.com/natcap/invest/issues/1859">#1859</a>).</li>
</ul>
</section>
<section id="sdr">
<h2>SDR</h2>
<ul>
<li>Raster outputs that previously contained per-pixel values (e.g., t/pixel) now contain per-hectare values (e.g., t/ha). (<a href="https://github.com/natcap/invest/issues/1270">InVEST #1270</a>).</li>
<li>D8 routing is now supported in addition to MFD (<a href="https://github.com/natcap/invest/issues/1440">#1440</a>).</li>
</ul>
</section>
<section id="seasonal-water-yield">
<h2>Seasonal Water Yield</h2>
<ul>
<li>D8 routing is now supported in addition to MFD (<a href="https://github.com/natcap/invest/issues/1440">#1440</a>).</li>
</ul>
</section>
<section id="urban-cooling">
<h2>Urban Cooling</h2>
<ul>
<li>Align rasters to the grid of the LULC raster, rather than the ET0 raster (<a href="https://github.com/natcap/invest/issues/1488">#1488</a>).</li>
<li>Updated the documentation for the <code>mean_t_air</code> attribute of the <code>buildings_with_stats.shp</code> output to clarify how the value is calculated. <a href="https://github.com/natcap/invest/issues/1746">https://github.com/natcap/invest/issues/1746</a></li>
<li>Fixed bug in the calculation of Cooling Capacity (CC) provided by parks, where the CC Index was not being properly incorporated. <a href="https://github.com/natcap/invest/issues/1726">https://github.com/natcap/invest/issues/1726</a></li>
</ul>
</section>
<section id="urban-stormwater-retention">
<h2>Urban Stormwater Retention</h2>
<ul>
<li>Fixed a bug causing <code>inf</code> values in volume outputs because nodata values were not being set correctly (<a href="https://github.com/natcap/invest/issues/1850">InVEST #1850</a>).</li>
</ul>
</section>
<section id="visitation-recreation-and-tourism">
<h2>Visitation: Recreation and Tourism</h2>
<ul>
<li>Added a database of geotagged tweets to support calculating twitter-user-days (TUD) as proxy for visitation rates. The model now calculates photo-user-days (PUD) and TUD and uses their average as the response variable in the regression model. Please refer to the User's Guide for more details on the regression model.</li>
<li>Output data were updated to support the new TUD results, and vector outputs are now in GeoPackage format instead of ESRI Shapefile.</li>
<li>Regression coefficients are still listed in a summary text file, and are now also included in a tabular output: "regression_coefficients.csv".</li>
</ul>
</section>
<section id="wind-energy">
<h2>Wind Energy</h2>
<ul>
<li>Fixed a bug that could cause the Workbench to crash when running the Wind Energy model with <code>Taskgraph</code> logging set to <code>DEBUG</code> (<a href="https://github.com/natcap/invest/issues/1497">InVEST #1497</a>).</li>
</ul>
</section>
</section>
<section id="section-2">
<h1>3.14.3 (2024-12-19)</h1>
<ul>
<li>
@ -147,7 +309,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-2">
<section id="section-3">
<h1>3.14.2 (2024-05-29)</h1>
<ul>
<li>
@ -268,7 +430,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-3">
<section id="section-4">
<h1>3.14.1 (2023-12-18)</h1>
<ul>
<li>
@ -399,7 +561,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-4">
<section id="section-5">
<h1>3.14.0 (2023-09-08)</h1>
<ul>
<li>
@ -604,7 +766,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-5">
<section id="section-6">
<h1>3.13.0 (2023-03-17)</h1>
<ul>
<li>
@ -737,7 +899,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-6">
<section id="section-7">
<h1>3.12.1 (2022-12-16)</h1>
<ul>
<li>
@ -810,7 +972,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-7">
<section id="section-8">
<h1>3.12.0 (2022-08-31)</h1>
<ul>
<li>
@ -934,7 +1096,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-8">
<section id="section-9">
<h1>3.11.0 (2022-05-24)</h1>
<ul>
<li>
@ -1032,7 +1194,7 @@ Everything else:
</li>
</ul>
</section>
<section id="section-9">
<section id="section-10">
<h1>3.10.2 (2022-02-08)</h1>
<ul>
<li>
@ -1134,7 +1296,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-10">
<section id="section-11">
<h1>3.10.1 (2022-01-06)</h1>
<ul>
<li>
@ -1149,7 +1311,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-11">
<section id="section-12">
<h1>3.10.0 (2022-01-04)</h1>
<ul>
<li>
@ -1333,7 +1495,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-12">
<section id="section-13">
<h1>3.9.2 (2021-10-29)</h1>
<ul>
<li>
@ -1394,7 +1556,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-13">
<section id="section-14">
<h1>3.9.1 (2021-09-22)</h1>
<ul>
<li>
@ -1543,7 +1705,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-14">
<section id="section-15">
<h1>3.9.0 (2020-12-11)</h1>
<ul>
<li>
@ -1731,7 +1893,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-15">
<section id="section-16">
<h1>3.8.9 (2020-09-15)</h1>
<ul>
<li>
@ -1756,7 +1918,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-16">
<section id="section-17">
<h1>3.8.8 (2020-09-04)</h1>
<ul>
<li>
@ -1856,7 +2018,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-17">
<section id="section-18">
<h1>3.8.7 (2020-07-17)</h1>
<ul>
<li>
@ -1895,7 +2057,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-18">
<section id="section-19">
<h1>3.8.6 (2020-07-03)</h1>
<ul>
<li>
@ -1910,7 +2072,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-19">
<section id="section-20">
<h1>3.8.5 (2020-06-26)</h1>
<ul>
<li>
@ -1961,7 +2123,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-20">
<section id="section-21">
<h1>3.8.4 (2020-06-05)</h1>
<ul>
<li>
@ -1996,7 +2158,7 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-21">
<section id="section-22">
<h1>3.8.3 (2020-05-29)</h1>
<ul>
<li>
@ -2011,13 +2173,13 @@ setuptools_scm</code> from the project root.</li>
</li>
</ul>
</section>
<section id="section-22">
<section id="section-23">
<h1>3.8.2 (2020-05-15)</h1>
<ul>
<li>InVEST's CSV encoding requirements are now described in the validation error message displayed when a CSV cannot be opened.</li>
</ul>
</section>
<section id="section-23">
<section id="section-24">
<h1>3.8.1 (2020-05-08)</h1>
<ul>
<li>Fixed a compilation issue on Mac OS X Catalina.</li>
@ -2041,7 +2203,7 @@ setuptools_scm</code> from the project root.</li>
<li>Update api-docs conf file to mock sdr.sdr_core and to use updated unittest mock</li>
</ul>
</section>
<section id="section-24">
<section id="section-25">
<h1>3.8.0 (2020-02-07)</h1>
<ul>
<li>Created a sub-directory for the sample data in the installation directory.</li>
@ -2097,7 +2259,7 @@ setuptools_scm</code> from the project root.</li>
<li>Added a new InVEST model: Urban Cooling Model.</li>
</ul>
</section>
<section id="section-25">
<section id="section-26">
<h1>3.7.0 (2019-05-09)</h1>
<ul>
<li>Refactoring Coastal Vulnerability (CV) model. CV now uses TaskGraph and Pygeoprocessing &gt;=1.6.1. The model is now largely vector-based instead of raster-based. Fewer input datasets are required for the same functionality. Runtime in sycnhronous mode is similar to previous versions, but runtime can be reduced with multiprocessing. CV also supports avoided recomputation for successive runs in the same workspace, even if a different file suffix is used. Output vector files are in CSV and geopackage formats.</li>
@ -2118,7 +2280,7 @@ setuptools_scm</code> from the project root.</li>
<li>Adding encoding='utf-8-sig' to pandas.read_csv() to support utils.build_lookup_from_csv() to read CSV files encoded with UTF-8 BOM (byte-order mark) properly.</li>
</ul>
</section>
<section id="section-26">
<section id="section-27">
<h1>3.6.0 (2019-01-30)</h1>
<ul>
<li>Correcting an issue with the InVEST Carbon Storage and Sequestration model where filepaths containing non-ASCII characters would cause the model's report generation to crash. The output report is now a UTF-8 document.</li>
@ -2148,7 +2310,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixing a case where a zero discount rate and rate of change in the carbon model would cause a divide by zero error.</li>
</ul>
</section>
<section id="section-27">
<section id="section-28">
<h1>3.5.0 (2018-08-14)</h1>
<ul>
<li>Bumped pygeoprocessing requirement to <code>pygeoprocessing&gt;=1.2.3</code>.</li>
@ -2170,7 +2332,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue in the model data of the crop production model where some crops were using incorrect climate bin rasters. Since the error was in the data and not the code, users will need to download the most recent version of InVEST's crop model data during the installation step to get the fix.</li>
</ul>
</section>
<section id="section-28">
<section id="section-29">
<h1>3.4.4 (2018-03-26)</h1>
<ul>
<li>InVEST now requires GDAL 2.0.0 and has been tested up to GDAL 2.2.3. Any API users of InVEST will need to use GDAL version &gt;= 2.0. When upgrading GDAL we noticed slight numerical differences in our test suite in both numerical raster differences, geometry transforms, and occasionally a single pixel difference when using <cite>gdal.RasterizeLayer</cite>. Each of these differences in the InVEST test suite is within a reasonable numerical tolerance and we have updated our regression test suite appropriately. Users comparing runs between previous versions of InVEST may also notice reasonable numerical differences between runs.</li>
@ -2180,7 +2342,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed a broken link to local and online user documentation from the Seasonal Water Yield model from the model's user interface.</li>
</ul>
</section>
<section id="section-29">
<section id="section-30">
<h1>3.4.3 (2018-03-26)</h1>
<ul>
<li>Fixed a critical issue in the carbon model UI that would incorrectly state the user needed a "REDD Priority Raster" when none was required.</li>
@ -2188,7 +2350,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue in wind energy UI that was incorrectly validating most of the inputs.</li>
</ul>
</section>
<section id="section-30">
<section id="section-31">
<h1>3.4.2 (2017-12-15)</h1>
<ul>
<li>Fixed a cross-platform issue with the UI where logfiles could not be dropped onto UI windows.</li>
@ -2198,7 +2360,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixing an issue with the <code>FileSystemRunDialog</code> where pressing the 'X' button in the corner of the window would close the window, but not reset its state. The window's state is now reset whenever the window is closed (and the window cannot be closed when the model is running)</li>
</ul>
</section>
<section id="section-31">
<section id="section-32">
<h1>3.4.1 (2017-12-11)</h1>
<ul>
<li>In the Coastal Blue Carbon model, the <code>interest_rate</code> parameter has been renamed to <code>inflation_rate</code>.</li>
@ -2206,7 +2368,7 @@ setuptools_scm</code> from the project root.</li>
<li>Added better error checking to the SDR model for missing <cite>ws_id</cite> and invalid <cite>ws_id</cite> values such as <cite>None</cite> or some non-integer value. Also added tests for the <cite>SDR</cite> validation module.</li>
</ul>
</section>
<section id="section-32">
<section id="section-33">
<h1>3.4.0 (2017-12-03)</h1>
<ul>
<li>Fixed an issue with most InVEST models where the suffix was not being reflected in the output filenames. This was due to a bug in the InVEST UI, where the suffix args key was assumed to be <code>'suffix'</code>. Instances of <code>InVESTModel</code> now accept a keyword argument to defined the suffix args key.</li>
@ -2238,7 +2400,7 @@ setuptools_scm</code> from the project root.</li>
<li>Updated the erodibility sample raster that ships with InVEST for the SDR model. The old version was in US units, in this version we convert to SI units as the model requires, and clipped the raster to the extents of the other stack to save disk space.</li>
</ul>
</section>
<section id="section-33">
<section id="section-34">
<h1>3.3.3 (2017-02-06)</h1>
<ul>
<li>Fixed an issue in the UI where the carbon model wouldn't accept negative numbers in the price increase of carbon.</li>
@ -2258,7 +2420,7 @@ setuptools_scm</code> from the project root.</li>
<li>Updated branding and usability of the InVEST installer for Windows, and the Mac Disk Image (.dmg).</li>
</ul>
</section>
<section id="section-34">
<section id="section-35">
<h1>3.3.2 (2016-10-17)</h1>
<ul>
<li>Partial test coverage for HRA model.</li>
@ -2297,7 +2459,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue in SDR that reported runtime overflow errors during normal processing even though the model completed without other errors.</li>
</ul>
</section>
<section id="section-35">
<section id="section-36">
<h1>3.3.1 (2016-06-13)</h1>
<ul>
<li>Refactored API documentation for readability, organization by relevant topics, and to allow docs to build on <a href="http://invest.readthedocs.io">invest.readthedocs.io</a>,</li>
@ -2323,7 +2485,7 @@ setuptools_scm</code> from the project root.</li>
<li>Updated Crop Production model to add a simplified UI, faster runtime, and more testing.</li>
</ul>
</section>
<section id="section-36">
<section id="section-37">
<h1>3.3.0 (2016-03-14)</h1>
<ul>
<li>Refactored Wind Energy model to use a CSV input for wind data instead of a Binary file.</li>
@ -2377,7 +2539,7 @@ setuptools_scm</code> from the project root.</li>
<li>Documentation to the GLOBIO code base including the large docstring for 'execute'.</li>
</ul>
</section>
<section id="section-37">
<section id="section-38">
<h1>3.2.0 (2015-05-31)</h1>
<p>InVEST 3.2.0 is a major release with the addition of several experimental models and tools as well as an upgrade to the PyGeoprocessing core:</p>
<ul>
@ -2390,11 +2552,11 @@ setuptools_scm</code> from the project root.</li>
<li>Miscelaneous performance patches and bug fixes.</li>
</ul>
</section>
<section id="section-38">
<section id="section-39">
<h1>3.1.3 (2015-04-23)</h1>
<p>InVEST 3.1.3 is a hotfix release patching a memory blocking issue resolved in PyGeoprocessing version 0.2.1. Users might have experienced slow runtimes on SDR or other routed models.</p>
</section>
<section id="section-39">
<section id="section-40">
<h1>3.1.2 (2015-04-15)</h1>
<p>InVEST 3.1.2 is a minor release patching issues mostly related to the freshwater routing models and signed GDAL Byte datasets.</p>
<ul>
@ -2410,7 +2572,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue in the Blue Carbon model that prevented the report from being generated in the outputs file.</li>
</ul>
</section>
<section id="section-40">
<section id="section-41">
<h1>3.1.1 (2015-03-13)</h1>
<p>InVEST 3.1.1 is a major performance and memory bug patch to the InVEST toolsuite. We recommend all users upgrade to this version.</p>
<ul>
@ -2433,7 +2595,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed a bug in Habitat Quality where the future output "quality_out_f.tif" was not reflecting the habitat value given in the sensitivity table for the specified landcover types.</li>
</ul>
</section>
<section id="section-41">
<section id="section-42">
<h1>3.1.0 (2014-11-19)</h1>
<p>InVEST 3.1.0 (<a href="http://www.naturalcapitalproject.org/download.html">http://www.naturalcapitalproject.org/download.html</a>) is a major software and science milestone that includes an overhauled sedimentation model, long awaited fixes to exponential decay routines in habitat quality and pollination, and a massive update to the underlying hydrological routing routines. The updated sediment model, called SDR (sediment delivery ratio), is part of our continuing effort to improve the science and capabilities of the InVEST tool suite. The SDR model inputs are backwards comparable with the InVEST 3.0.1 sediment model with two additional global calibration parameters and removed the need for the retention efficiency parameter in the biophysical table; most users can run SDR directly with the data they have prepared for previous versions. The biophysical differences between the models are described in a section within the SDR user's guide and represent a superior representation of the hydrological connectivity of the watershed, biophysical parameters that are independent of cell size, and a more accurate representation of sediment retention on the landscape. Other InVEST improvements to include standard bug fixes, performance improvements, and usability features which in part are described below:</p>
<ul>
@ -2474,7 +2636,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue where the data type of the nodata value in a raster might be different than the values in the raster. This was common in the case of 64 bit floating point values as nodata when the underlying raster was 32 bit. Now nodata values are cast to the underlying types which improves the reliability of many of the InVEST models.</li>
</ul>
</section>
<section id="section-42">
<section id="section-43">
<h1>3.0.1 (2014-05-19)</h1>
<ul>
<li>Blue Carbon model released.</li>
@ -2496,7 +2658,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed an issue in Marine Water Quality where the UV points were supposed to be optional, but instead raised an exception when not passed in.</li>
</ul>
</section>
<section id="section-43">
<section id="section-44">
<h1>3.0.0 (2014-03-23)</h1>
<p>The 3.0.0 release of InVEST represents a shift away from the ArcGIS to the InVEST standalone computational platform. The only exception to this shift is the marine coastal protection tier 1 model which is still supported in an ArcGIS toolbox and has no InVEST 3.0 standalone at the moment. Specific changes are detailed below</p>
<ul>
@ -2515,7 +2677,7 @@ setuptools_scm</code> from the project root.</li>
<li>Numerous other minor bug fixes and performance enhacnements.</li>
</ul>
</section>
<section id="section-44">
<section id="section-45">
<h1>2.6.0 (2013-12-16)</h1>
<p>The 2.6.0 release of InVEST removes most of the old InVEST models from the Arc toolbox in favor of the new InVEST standalone models. While we have been developing standalone equivalents for the InVEST Arc models since version 2.3.0, this is the first release in which we removed support for the deprecated ArcGIS versions after an internal review of correctness, performance, and stability on the standalones. Additionally, this is one of the last milestones before the InVEST 3.0.0 release later next year which will transition InVEST models away from strict ArcGIS dependence to a standalone form.</p>
<p>Specifically, support for the following models have been moved from the ArcGIS toolbox to their Windows based standalones: (1) hydropower/water yield, (2) finfish aquaculture, (3) coastal protection tier 0/coastal vulnerability, (4) wave energy, (5) carbon, (6) habitat quality/biodiversity, (7) pollination, (8) timber, and (9) overlap analysis. Additionally, documentation references to ArcGIS for those models have been replaced with instructions for launching standalone InVEST models from the Windows start menu.</p>
@ -2538,7 +2700,7 @@ setuptools_scm</code> from the project root.</li>
<li>Changing support from <a href="mailto:richsharp@stanford.edu">richsharp@stanford.edu</a> to the user support forums at <a href="http://ncp-yamato.stanford.edu/natcapforums">http://ncp-yamato.stanford.edu/natcapforums</a>.</li>
</ul>
</section>
<section id="section-45">
<section id="section-46">
<h1>2.5.6 (2013-09-06)</h1>
<p>The 2.5.6 release of InVEST that addresses minor bugs, performance tweaks, and new functionality of the InVEST standalone models. Including:</p>
<ul>
@ -2564,7 +2726,7 @@ setuptools_scm</code> from the project root.</li>
<li>Added an infrastructure feature so that temporary files are created in the user's workspace rather than at the system level folder. This lets users work in a secondary workspace on a USB attached hard drive and use the space of that drive, rather than the primary operating system drive.</li>
</ul>
</section>
<section id="section-46">
<section id="section-47">
<h1>2.5.5 (2013-08-06)</h1>
<p>The 2.5.5 release of InVEST that addresses minor bugs, performance tweaks, and new functionality of the InVEST standalone models. Including:</p>
<blockquote>
@ -2658,7 +2820,7 @@ setuptools_scm</code> from the project root.</li>
</ul>
</blockquote>
</section>
<section id="section-47">
<section id="section-48">
<h1>2.5.4 (2013-06-07)</h1>
<p>This is a minor release of InVEST that addresses numerous minor bugs and performance tweaks in the InVEST 3.0 models. Including:</p>
<blockquote>
@ -2708,15 +2870,15 @@ setuptools_scm</code> from the project root.</li>
</ul>
</blockquote>
</section>
<section id="section-48">
<section id="section-49">
<h1>2.5.3 (2013-03-21)</h1>
<p>This is a minor release of InVEST that fixes an issue with the HRA model that caused ArcGIS versions of the model to fail when calculating habitat maps for risk hotspots. This upgrade is strongly recommended for users of InVEST 2.5.1 or 2.5.2.</p>
</section>
<section id="section-49">
<section id="section-50">
<h1>2.5.2 (2013-03-17)</h1>
<p>This is a minor release of InVEST that fixes an issue with the HRA sample data that caused ArcGIS versions of the model to fail on the training data. There is no need to upgrade for most users unless you are doing InVEST training.</p>
</section>
<section id="section-50">
<section id="section-51">
<h1>2.5.1 (2013-03-12)</h1>
<p>This is a minor release of InVEST that does not add any new models, but does add additional functionality, stability, and increased performance to one of the InVEST 3.0 standalones:</p>
<blockquote>
@ -2735,7 +2897,7 @@ setuptools_scm</code> from the project root.</li>
</blockquote>
<p>Additionally, this minor release fixes a bug in the InVEST user interface where collapsible containers became entirely non-interactive.</p>
</section>
<section id="section-51">
<section id="section-52">
<h1>2.5.0 (2013-03-08)</h1>
<p>This a major release of InVEST that includes new standalone versions (ArcGIS is not required) our models as well as additional functionality, stability, and increased performance to many of the existing models. This release is timed to support our group's annual training event at Stanford University. We expect to release InVEST 2.5.1 a couple of weeks after to address any software issues that arise during the training. See the release notes below for details of the release, and please contact <a href="mailto:richsharp@stanford.edu">richsharp@stanford.edu</a> for any issues relating to software:</p>
<blockquote>
@ -2822,7 +2984,7 @@ setuptools_scm</code> from the project root.</li>
</blockquote>
</blockquote>
</section>
<section id="section-52">
<section id="section-53">
<h1>2.4.5 (2013-02-01)</h1>
<p>This is a minor release of InVEST that does not add any new models, but does add additional functionality, stability, and increased performance to many of the InVEST 3.0 standalones:</p>
<blockquote>
@ -2882,7 +3044,7 @@ setuptools_scm</code> from the project root.</li>
</ul>
</blockquote>
</section>
<section id="section-53">
<section id="section-54">
<h1>2.4.4 (2012-10-24)</h1>
<ul>
<li>Fixes memory errors experienced by some users in the Carbon Valuation 3.0 Beta model.</li>
@ -2890,7 +3052,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixes an issue importing packages for some officially-unreleased InVEST models.</li>
</ul>
</section>
<section id="section-54">
<section id="section-55">
<h1>2.4.3 (2012-10-19)</h1>
<ul>
<li>Fixed a minor issue with hydropower output vaulation rasters whose statistics were not pre-calculated. This would cause the range in ArcGIS to show ther rasters at -3e38 to 3e38.</li>
@ -2901,20 +3063,20 @@ setuptools_scm</code> from the project root.</li>
<li>Added a feature to all InVEST 3.0 models to list disk usage before and after each run and in most cases report a low free space error if relevant.</li>
</ul>
</section>
<section id="section-55">
<section id="section-56">
<h1>2.4.2 (2012-10-15)</h1>
<ul>
<li>Fixed an issue with the ArcMap document where the paths to default data were not saved as relative paths. This caused the default data in the document to not be found by ArcGIS.</li>
<li>Introduced some more memory-efficient processing for Biodiversity 3.0 Beta. This fixes an out-of-memory issue encountered by some users when using very large raster datasets as inputs.</li>
</ul>
</section>
<section id="section-56">
<section id="section-57">
<h1>2.4.1 (2012-10-08)</h1>
<ul>
<li>Fixed a compatibility issue with ArcGIS 9.3 where the ArcMap and ArcToolbox were unable to be opened by Arc 9.3.</li>
</ul>
</section>
<section id="section-57">
<section id="section-58">
<h1>2.4.0 (2012-10-05)</h1>
<p>Changes in InVEST 2.4.0</p>
<p>General:</p>
@ -2994,7 +3156,7 @@ setuptools_scm</code> from the project root.</li>
<li>Fixed a visualization bug with wave energy where output rasters did not have the min/max/stdev calculations on them. This made the default visualization in arc be a gray blob.</li>
</ul>
</section>
<section id="section-58">
<section id="section-59">
<h1>2.3.0 (2012-08-02)</h1>
<p>Changes in InVEST 2.3.0</p>
<p>General:</p>
@ -3060,7 +3222,7 @@ setuptools_scm</code> from the project root.</li>
<li>Other minor bug fixes and runtime performance tweaks in the 3.0 framework.</li>
</ul>
</section>
<section id="section-59">
<section id="section-60">
<h1>2.2.2 (2012-03-03)</h1>
<p>Changes in InVEST 2.2.2</p>
<p>General:</p>
@ -3081,14 +3243,14 @@ setuptools_scm</code> from the project root.</li>
<dd>toolbox if the workspace name is too long.</dd>
</dl>
</section>
<section id="section-60">
<section id="section-61">
<h1>2.2.1 (2012-01-26)</h1>
<p>Changes in InVEST 2.2.1</p>
<p>General:</p>
<p>This is a minor release which fixes the following defects:</p>
<p>-A variety of miscellaneous bugs were fixed that were causing crashes of the Coastal Protection model in Arc 9.3. -Fixed an issue in the Pollination model that was looking for an InVEST1005 directory. -The InVEST "models only" release had an entry for the InVEST 3.0 Beta tools, but was missing the underlying runtime. This has been added to the models only 2.2.1 release at the cost of a larger installer. -The default InVEST ArcMap document wouldn't open in ArcGIS 9.3. It can now be opened by Arc 9.3 and above. -Minor updates to the Coastal Protection user's guide.</p>
</section>
<section id="section-61">
<section id="section-62">
<h1>2.2.0 (2011-12-22)</h1>
<p>In this release we include updates to the habitat risk assessment model, updates to Coastal Vulnerability Tier 0 (previously named Coastal Protection), and a new tier 1 Coastal Vulnerability tool. Additionally, we are releasing a beta version of our 3.0 platform that includes the terrestrial timber and carbon models.</p>
<p>See the "Marine Models" and "InVEST 3.0 Beta" sections below for more details.</p>
@ -3142,7 +3304,7 @@ setuptools_scm</code> from the project root.</li>
</dd>
</dl>
</section>
<section id="section-62">
<section id="section-63">
<h1>2.1.1 (2011-10-17)</h1>
<p>Changes in InVEST 2.1.1</p>
<p>General:</p>

2
workbench/package.json
View File

@ -1,6 +1,6 @@
{
"name": "invest-workbench",
"version": "3.14.3",
"version": "3.15.0",
"description": "Models that map and value the goods and services from nature that sustain and fulfill human life",
"main": "build/main/main.js",
"homepage": "./",

11
workbench/src/renderer/components/Changelog/index.jsx
View File

@ -5,6 +5,7 @@ import Modal from 'react-bootstrap/Modal';
import { MdClose } from 'react-icons/md';
import { useTranslation } from 'react-i18next';
import { openLinkInBrowser } from '../../utils';
import pkg from '../../../../package.json';
import { ipcMainChannels } from '../../../main/ipcMainChannels';
@ -29,8 +30,10 @@ export default function Changelog(props) {
// Find the section whose heading explicitly matches the current version.
const versionStr = pkg.version;
const escapedVersionStr = versionStr.split('.').join('\\.');
// Find section with h1 matching current version,
// and get everything up to (but not including) the next h1.
const sectionRegex = new RegExp(
`<section.*?>[\\s]*?<h1>${escapedVersionStr}\\b[\\s\\S]*?</h1>[\\s\\S]*?</section>`
`<section.*?>[\\s]*?<h1>${escapedVersionStr}\\b[\\s\\S]*?</h1>[\\s\\S]*?<h1`
);
const sectionMatches = htmlString.match(sectionRegex);
if (sectionMatches && sectionMatches.length) {
@ -55,12 +58,6 @@ export default function Changelog(props) {
// Once HTML content has loaded, set up links to open in browser
// (instead of in an Electron window).
useEffect(() => {
const openLinkInBrowser = (event) => {
event.preventDefault();
ipcRenderer.send(
ipcMainChannels.OPEN_EXTERNAL_URL, event.currentTarget.href
);
};
document.querySelectorAll('.link-external').forEach(link => {
link.addEventListener('click', openLinkInBrowser);
});

54
workbench/src/renderer/components/LogTab/index.jsx
View File

@ -1,4 +1,4 @@
import React, { useEffect, useRef } from 'react';
import React, { useEffect, useRef, useState } from 'react';
import ReactDom from 'react-dom';
import PropTypes from 'prop-types';
@ -13,15 +13,65 @@ const { ipcRenderer } = window.Workbench.electron;
function LogDisplay(props) {
const ref = useRef();
const [autoScroll, setAutoScroll] = useState(true);
const [prevScrollTop, setPrevScrollTop] = useState(0);
// A scroll event doesn't tell us whether it was initiated by a user or by code,
// so we assume all scroll events are user-initiated unless otherwise specified.
const [userInitiatedScroll, setUserInitiatedScroll] = useState(true);
let scrollHandlerTimer;
// `scrollOffsetThreshold` is used to determine when user has scrolled to bottom of window.
// It includes a buffer to account for imprecision stemming from rounding errors and/or
// scroll event throttling. 24px (the height of one line of log output) seems to be
// large enough to detect scroll events we want, and small enough to avoid false positives.
const scrollOffsetThreshold = 24;
useEffect(() => {
ref.current.scrollTop = ref.current.scrollHeight;
if (autoScroll) {
// Setting `ref.current.scrollTop` will fire a scroll event, which will
// result in a call to `handleScroll`. To avoid unnecessary operations in
// `handleScroll`, we flag the next scroll event as _not_ user-initiated.
setUserInitiatedScroll(false);
ref.current.scrollTop = ref.current.scrollHeight;
}
}, [props.logdata]);
// Check scroll direction or position IFF scroll event was user-initiated.
// Always update `prevScrollTop` and reset `userInitiatedScroll` to `true`.
const handleScroll = () => {
if (scrollHandlerTimer) {
clearTimeout(scrollHandlerTimer);
}
scrollHandlerTimer = setTimeout(() => {
const currentScrollTop = ref.current.scrollTop;
if (userInitiatedScroll) {
if (autoScroll) {
// If user has scrolled up, halt auto-scrolling.
const scrollingUp = (currentScrollTop < prevScrollTop);
if (scrollingUp) {
setAutoScroll(false);
}
} else {
// If user has scrolled back to the bottom, resume auto-scrolling.
const currentScrollOffset = ref.current.scrollHeight - currentScrollTop;
if (Math.abs(ref.current.offsetHeight - currentScrollOffset) <= scrollOffsetThreshold) {
setAutoScroll(true);
}
}
}
setPrevScrollTop(currentScrollTop);
setUserInitiatedScroll(true);
}, 10);
};
return (
<Col
className="text-break"
id="log-display"
ref={ref}
onScroll={handleScroll}
>
{
props.logdata.map(([line, cls], idx) => (

12
workbench/src/renderer/components/ResourcesLinks/index.jsx
View File

@ -4,6 +4,7 @@ import PropTypes from 'prop-types';
import { MdOpenInNew } from 'react-icons/md';
import { useTranslation } from 'react-i18next';
import { openLinkInBrowser } from '../../utils';
import { ipcMainChannels } from '../../../main/ipcMainChannels';
const { ipcRenderer } = window.Workbench.electron;
@ -39,15 +40,6 @@ const FORUM_TAGS = {
wind_energy: 'wind-energy',
};
/**
* Open the target href in the default web browser.
*/
function handleForumClick(event) {
event.preventDefault();
ipcRenderer.send(
ipcMainChannels.OPEN_EXTERNAL_URL, event.currentTarget.href
);
}
/**
* Open the target href in an electron window.
@ -92,7 +84,7 @@ export default function ResourcesTab(props) {
href={forumURL}
title={forumURL}
aria-label="go to frequently asked questions in web browser"
onClick={handleForumClick}
onClick={openLinkInBrowser}
>
<MdOpenInNew className="mr-1" />
{t("Frequently Asked Questions")}

191
workbench/src/renderer/components/SettingsModal/MetadataForm/index.jsx Normal file
View File

@ -0,0 +1,191 @@
import React, { useState, useEffect } from 'react';
import { useTranslation } from 'react-i18next';
import Row from 'react-bootstrap/Row';
import Col from 'react-bootstrap/Col';
import Alert from 'react-bootstrap/Alert';
import Button from 'react-bootstrap/Button';
import Form from 'react-bootstrap/Form';
import {
getGeoMetaMakerProfile,
setGeoMetaMakerProfile,
} from '../../../server_requests';
import { openLinkInBrowser } from '../../../utils';
function AboutMetadataDiv() {
const { t } = useTranslation();
return (
<div>
<h5>{t('Metadata for InVEST results')}</h5>
<p>
{t(`InVEST models create metadata files that describe each dataset
created by the model. These are the "*.yml", or YAML, files
in the output workspace after running a model.`)}
</p>
<p>
{t(`Open a YAML file in a text editor to read the metadata and even add
to it. Metadata includes descriptions of fields in tables,
the bands in a raster, and other useful information.`)}
</p>
<p>
{t(`Some properties of the metadata are configurable here. You may
save information about the data author (you) and data license
information. These details are included in all metadata documents
created by InVEST and by GeoMetaMaker. This information is optional,
it never leaves your computer unless you share your data and metadata,
and you may modify it here anytime.`)}
</p>
<p>
{t('InVEST uses GeoMetaMaker to generate metadata. Learn more about ')}
<a
href="https://github.com/natcap/geometamaker"
onClick={openLinkInBrowser}
>GeoMetaMaker on Github</a>.
</p>
</div>
);
}
function FormRow(label, value, handler) {
return (
<Row>
<Col sm="4">
<Form.Label>{label}</Form.Label>
</Col>
<Col sm="8">
<Form.Control
type="text"
value={value || ''}
onChange={(e) => handler(e.currentTarget.value)}
/>
</Col>
</Row>
);
}
/**
* A form for submitting GeoMetaMaker profile data.
*/
export default function MetadataForm() {
const { t } = useTranslation();
const [contactName, setContactName] = useState('');
const [contactEmail, setContactEmail] = useState('');
const [contactOrg, setContactOrg] = useState('');
const [contactPosition, setContactPosition] = useState('');
const [licenseTitle, setLicenseTitle] = useState('');
const [licenseURL, setLicenseURL] = useState('');
const [alertMsg, setAlertMsg] = useState('');
const [alertError, setAlertError] = useState(false);
const [showInfo, setShowInfo] = useState(false);
useEffect(() => {
async function loadProfile() {
const profile = await getGeoMetaMakerProfile();
if (profile && profile.contact) {
setContactName(profile.contact.individual_name);
setContactEmail(profile.contact.email);
setContactOrg(profile.contact.organization);
setContactPosition(profile.contact.position_name);
}
if (profile && profile.license) {
setLicenseTitle(profile.license.title);
setLicenseURL(profile.license.path);
}
}
loadProfile();
}, []);
const handleSubmit = async (event) => {
event.preventDefault();
const { message, error } = await setGeoMetaMakerProfile({
contact: {
individual_name: contactName,
email: contactEmail,
organization: contactOrg,
position_name: contactPosition,
},
license: {
title: licenseTitle,
path: licenseURL,
},
});
setAlertMsg(message);
setAlertError(error);
};
/**
* A change handler for the form, not for individual fields
*/
const handleChange = async () => {
// Clear the alert message only if it's not an error message
if (alertError) { return; }
setAlertMsg('');
};
return (
<div id="metadata-form">
{
(showInfo)
? <AboutMetadataDiv />
: (
<Form onSubmit={handleSubmit} onChange={handleChange}>
<fieldset>
<legend>{t('Contact Information')}</legend>
<Form.Group controlId="name">
{FormRow(t('Full name'), contactName, setContactName)}
</Form.Group>
<Form.Group controlId="email">
{FormRow(t('Email address'), contactEmail, setContactEmail)}
</Form.Group>
<Form.Group controlId="job-title">
{FormRow(t('Job title'), contactPosition, setContactPosition)}
</Form.Group>
<Form.Group controlId="organization">
{FormRow(t('Organization name'), contactOrg, setContactOrg)}
</Form.Group>
</fieldset>
<fieldset>
<legend>{t('Data License Information')}</legend>
<Form.Group controlId="license-title">
{FormRow(t('Title'), licenseTitle, setLicenseTitle)}
</Form.Group>
<Form.Group controlId="license-url">
{FormRow('URL', licenseURL, setLicenseURL)}
</Form.Group>
</fieldset>
<Form.Row>
<Button
type="submit"
variant="primary"
className="my-1 py2 mx-2"
>
{t('Save Metadata')}
</Button>
{
(alertMsg) && (
<Alert
className="my-1 py-2"
variant={alertError ? 'danger' : 'success'}
>
{alertMsg}
</Alert>
)
}
</Form.Row>
</Form>
)
}
<Button
variant="outline-secondary"
className="my-1 py-2 mx-2 info-toggle"
onClick={() => setShowInfo((prevState) => !prevState)}
>
{showInfo ? t('Hide Info') : t('More Info')}
</Button>
</div>
);
}

28
workbench/src/renderer/components/SettingsModal/index.jsx
View File

@ -11,13 +11,13 @@ import {
MdSettings,
MdClose,
MdTranslate,
MdWarningAmber,
} from 'react-icons/md';
import { BsChevronExpand } from 'react-icons/bs';
import { BsChevronDown } from 'react-icons/bs';
import { withTranslation } from 'react-i18next';
import { ipcMainChannels } from '../../../main/ipcMainChannels';
import { getSupportedLanguages } from '../../server_requests';
import MetadataForm from './MetadataForm';
const { ipcRenderer } = window.Workbench.electron;
@ -184,6 +184,7 @@ class SettingsModal extends React.Component {
</Form.Group>
) : <React.Fragment />
}
<hr />
<Form.Group as={Row}>
<Form.Label column sm="6" htmlFor="logging-select">
{t('Logging threshold')}
@ -250,7 +251,7 @@ class SettingsModal extends React.Component {
eventKey="0"
className="pt-0"
>
<BsChevronExpand className="mx-1" />
<BsChevronDown className="mx-1" />
<span className="small"><u>{t('more info')}</u></span>
</Accordion.Toggle>
<Accordion.Collapse eventKey="0" className="pr-1">
@ -275,7 +276,7 @@ class SettingsModal extends React.Component {
<Button
variant="primary"
onClick={this.switchToDownloadModal}
className="w-50"
className="w-100"
>
{t('Download Sample Data')}
</Button>
@ -283,11 +284,26 @@ class SettingsModal extends React.Component {
<Button
variant="secondary"
onClick={clearJobsStorage}
className="mr-2 w-50"
className="mr-2 w-100"
>
{t('Clear Recent Jobs')}
</Button>
<span>{t('no invest workspaces will be deleted')}</span>
<span><em>{t('*no invest workspaces will be deleted')}</em></span>
<hr />
<Accordion>
<Accordion.Toggle
as={Button}
variant="outline-secondary"
eventKey="0"
className="mr-2 w-100"
>
{t('Configure Metadata')}
<BsChevronDown className="mx-1" />
</Accordion.Toggle>
<Accordion.Collapse eventKey="0">
<MetadataForm />
</Accordion.Collapse>
</Accordion>
</Modal.Body>
</Modal>
{

18
workbench/src/renderer/components/SetupTab/ArgInput/index.jsx
View File

@ -1,4 +1,4 @@
import React, { useEffect, useRef, useState } from 'react';
import React, { useEffect, useId, useRef, useState } from 'react';
import PropTypes from 'prop-types';
import { useTranslation } from 'react-i18next';
@ -41,7 +41,7 @@ function filterSpatialOverlapFeedback(message, filepath) {
function FormLabel(props) {
const {
argkey, argname, argtype, required, units,
inputId, argkey, argname, argtype, required, units,
} = props;
const userFriendlyArgType = parseArgType(argtype);
@ -49,7 +49,7 @@ function FormLabel(props) {
const includeComma = userFriendlyArgType && optional;
return (
<Form.Label column sm="3" htmlFor={argkey}>
<Form.Label column sm="3" htmlFor={inputId}>
<span className="argname">{argname} </span>
{
(userFriendlyArgType || optional) &&
@ -157,6 +157,7 @@ function parseArgType(argtype) {
}
export default function ArgInput(props) {
const uniqueId = useId();
const inputRef = useRef();
const {
@ -176,6 +177,8 @@ export default function ArgInput(props) {
} = props;
let { validationMessage } = props;
const inputId = `${argkey}-${uniqueId}`;
// Occasionaly we want to force a scroll to the end of input fields
// so that the most important part of a filepath is visible.
// scrollEventCount changes on drop events and on use of the browse button.
@ -221,7 +224,7 @@ export default function ArgInput(props) {
<Button
aria-label={`browse for ${argSpec.name}`}
className="ml-2"
id={argkey}
id={inputId}
variant="outline-dark"
value={argSpec.type} // dialog will limit options accordingly
name={argkey}
@ -242,7 +245,7 @@ export default function ArgInput(props) {
<Form.Check
inline
type="switch"
id={argkey}
id={inputId}
name={argkey}
checked={value}
onChange={() => updateArgValues(argkey, !value)}
@ -253,7 +256,7 @@ export default function ArgInput(props) {
} else if (argSpec.type === 'option_string') {
form = (
<Form.Control
id={argkey}
id={inputId}
as="select"
name={argkey}
value={value}
@ -279,7 +282,7 @@ export default function ArgInput(props) {
<React.Fragment>
<Form.Control
ref={inputRef}
id={argkey}
id={inputId}
name={argkey}
type="text"
placeholder={placeholderText}
@ -309,6 +312,7 @@ export default function ArgInput(props) {
className={className} // this grays out the label but doesn't actually disable the field
>
<FormLabel
inputId={inputId}
argkey={argkey}
argname={argSpec.name}
argtype={argSpec.type}

19
workbench/src/renderer/menubar/about.jsx
View File

@ -97,13 +97,13 @@ ReactDom.render(
<table>
<tbody>
<tr>
<td>PyInstaller</td>
<td>GPL</td>
<td>geometamaker</td>
<td>Apache 2.0</td>
<td>
<a
href="http://pyinstaller.org"
href="https://github.com/natcap/geometamaker"
>
http://pyinstaller.org
https://github.com/natcap/geometamaker
</a>
</td>
</tr>
@ -140,6 +140,17 @@ ReactDom.render(
</a>
</td>
</tr>
<tr>
<td>PyInstaller</td>
<td>GPL</td>
<td>
<a
href="http://pyinstaller.org"
>
http://pyinstaller.org
</a>
</td>
</tr>
<tr>
<td>rtree</td>
<td>MIT</td>

52
workbench/src/renderer/server_requests.js
View File

@ -204,3 +204,55 @@ export async function getSupportedLanguages() {
.catch((error) => logger.error(error.stack))
);
}
/**
* Get the user-profile from GeoMetaMaker.
*
* @returns {Promise} resolves object
*/
export async function getGeoMetaMakerProfile() {
return (
window.fetch(`${HOSTNAME}:${PORT}/${PREFIX}/get_geometamaker_profile`, {
method: 'get',
})
.then((response) => response.json())
.catch((error) => logger.error(error.stack))
);
}
/**
* Set the user-profile in GeoMetaMaker.
*
* @param {object} payload {
* contact: {
* individual_name: string
* email: string
* organization: string
* position_name: string
* }
* license: {
* title: string
* url: string
* }
* }
* @returns {Promise} resolves object
*/
export async function setGeoMetaMakerProfile(payload) {
return (
window.fetch(`${HOSTNAME}:${PORT}/${PREFIX}/set_geometamaker_profile`, {
method: 'post',
body: JSON.stringify(payload),
headers: { 'Content-Type': 'application/json' },
})
.then((response) => response.json())
.then(({ message, error }) => {
if (error) {
logger.error(message);
} else {
logger.debug(message);
}
return { message, error };
})
.catch((error) => logger.error(error.stack))
);
}

42
workbench/src/renderer/styles/style.css
View File

@ -596,6 +596,44 @@ input[type=text]::placeholder {
max-width: 600px;
}
.settings-modal .accordion:has(.show) button svg {
transform: rotate(180deg);
}
.settings-modal .accordion button {
display: flex;
align-items: center;
justify-content: space-between;
}
#metadata-form {
/*This div toggles between a form and some help text.
position and static height ensure no resizing to content.*/
position: relative;
height: 460px;
background-color: ghostwhite;
border: dotted;
border-width: thin;
padding: 0.5rem;
}
/* This button should also not move as content is toggled*/
#metadata-form .info-toggle {
position: absolute;
bottom: 10px;
right: 10px;
}
#metadata-form legend {
font-size: 1.1rem;
text-decoration-line: underline;
}
#metadata-form .form-group {
padding-left: 1.5rem;
padding-right: 1.5rem;
}
/* Save As modal */
.save-as-modal svg {
margin-bottom: 0.2rem;
@ -632,3 +670,7 @@ input[type=text]::placeholder {
background-size: 1rem 1rem;
}
}
code {
color: inherit;
}

28
workbench/src/renderer/ui_config.js
View File

@ -68,27 +68,22 @@ const UI_SPEC = {
carbon: {
order: [
['workspace_dir', 'results_suffix'],
['lulc_cur_path', 'carbon_pools_path'],
['calc_sequestration', 'lulc_fut_path'],
['do_redd', 'lulc_redd_path'],
['lulc_bas_path', 'carbon_pools_path'],
['calc_sequestration', 'lulc_alt_path'],
[
'do_valuation',
'lulc_cur_year',
'lulc_fut_year',
'lulc_bas_year',
'lulc_alt_year',
'price_per_metric_ton_of_c',
'discount_rate',
'rate_change',
],
],
enabledFunctions: {
lulc_fut_path: isSufficient.bind(null, 'calc_sequestration'),
do_redd: isSufficient.bind(null, 'calc_sequestration'),
lulc_redd_path: isSufficient.bind(null, 'do_redd'),
lulc_alt_path: isSufficient.bind(null, 'calc_sequestration'),
do_valuation: isSufficient.bind(null, 'calc_sequestration'),
lulc_cur_year: isSufficient.bind(null, 'do_valuation'),
lulc_fut_year: isSufficient.bind(null, 'do_valuation'),
lulc_bas_year: isSufficient.bind(null, 'do_valuation'),
lulc_alt_year: isSufficient.bind(null, 'do_valuation'),
price_per_metric_ton_of_c: isSufficient.bind(null, 'do_valuation'),
discount_rate: isSufficient.bind(null, 'do_valuation'),
rate_change: isSufficient.bind(null, 'do_valuation'),
@ -202,7 +197,7 @@ const UI_SPEC = {
['dem_path', 'lulc_path', 'runoff_proxy_path', 'watersheds_path', 'biophysical_table_path'],
['calc_p'],
['calc_n', 'subsurface_critical_length_n', 'subsurface_eff_n'],
['threshold_flow_accumulation', 'k_param'],
['flow_dir_algorithm', 'threshold_flow_accumulation', 'k_param', 'runoff_proxy_av'],
],
enabledFunctions: {
subsurface_critical_length_n: isSufficient.bind(null, 'calc_n'),
@ -219,7 +214,8 @@ const UI_SPEC = {
recreation: {
order: [
['workspace_dir', 'results_suffix'],
['aoi_path', 'start_year', 'end_year'],
['aoi_path'],
['start_year', 'end_year'],
['compute_regression', 'predictor_table_path', 'scenario_predictor_table_path'],
['grid_aoi', 'grid_type', 'cell_size'],
],
@ -286,7 +282,7 @@ const UI_SPEC = {
['dem_path', 'erosivity_path', 'erodibility_path'],
['lulc_path', 'biophysical_table_path'],
['watersheds_path', 'drainage_path'],
['threshold_flow_accumulation', 'k_param', 'sdr_max', 'ic_0_param', 'l_max'],
['flow_dir_algorithm', 'threshold_flow_accumulation', 'k_param', 'sdr_max', 'ic_0_param', 'l_max'],
],
},
seasonal_water_yield: {
@ -294,7 +290,7 @@ const UI_SPEC = {
['workspace_dir', 'results_suffix'],
['lulc_raster_path', 'biophysical_table_path'],
['dem_raster_path', 'aoi_path'],
['threshold_flow_accumulation', 'beta_i', 'gamma'],
['flow_dir_algorithm', 'threshold_flow_accumulation', 'beta_i', 'gamma'],
['user_defined_local_recharge', 'l_path', 'et0_dir', 'precip_dir', 'soil_group_path'],
['monthly_alpha', 'alpha_m', 'monthly_alpha_path'],
['user_defined_climate_zones', 'rain_events_table_path', 'climate_zone_table_path', 'climate_zone_raster_path'],

Some files were not shown because too many files have changed in this diff Show More