VeniVidiVici
/
invest
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updates to various scripts for python3 compatibility. RE:#BITBUCKET-3905

This commit is contained in:
James Douglass 2019-09-16 12:13:20 -07:00
parent 56ac8f09d4
commit 4c10b6bea6
6 changed files with 84 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
installer/darwin/dmgconf.py
View File

@ -10,8 +10,8 @@ def get_size(start_path = '.'):
return str(int(total_size/1024.) + 1024*50) + 'K'
size = get_size(defines['investdir'])
print 'Volume size: %s' % size
print 'Packaging dirname %s' % defines['investdir']
print('Volume size: %s' % size)
print('Packaging dirname %s' % defines['investdir'])
_invest_dirname = os.path.basename(defines['investdir'])
badge_icon = os.path.join(CWD, 'invest.icns')

4
scripts/convert-requirements-to-conda-yml.py
View File

@ -85,9 +85,9 @@ def build_environment_from_requirements(cli_args):
pip_deps_string = '- pip:\n' + '\n'.join([' - %s' % dep for dep in
sorted(pip_requirements,
key=lambda x: x.lower())])
print YML_TEMPLATE.format(
print(YML_TEMPLATE.format(
conda_dependencies=conda_deps_string,
pip_dependencies=pip_deps_string)
pip_dependencies=pip_deps_string))
if __name__ == '__main__':

15
scripts/fetch_usage_polygon.py
View File

@ -1,7 +1,6 @@
"""Script to generate shapefiles from InVEST logging database."""
import urllib
import urllib2
import urllib.request
import datetime
import json
@ -12,16 +11,16 @@ _ENDPOINTS_INDEX_URL = (
if __name__ == '__main__':
USAGE_POLYGON_URL = json.loads(urllib.urlopen(
USAGE_POLYGON_URL = json.loads(urllib.request.urlopen(
_ENDPOINTS_INDEX_URL).read().strip())['STATS']
OUT_FILENAME = 'invest_usage_%s.geojson' % (
datetime.datetime.now().isoformat('_').replace(':', '_'))
print 'Writing usage to %s' % OUT_FILENAME
print('Writing usage to %s' % OUT_FILENAME)
with open(OUT_FILENAME, 'w') as out_geojson:
print 'downloading run_summary vector'
print('downloading run_summary vector')
out_geojson.write(
urllib2.urlopen(urllib2.Request(
USAGE_POLYGON_URL)).read())
print 'Done.'
urllib.request.urlopen(urllib.request.Request(
USAGE_POLYGON_URL)).read().decode('utf-8'))
print('Done.')

68
scripts/wave_energy_scripts/wave_csv_to_points.py
View File

@ -1,38 +1,40 @@
"""Python module that creates a point shapefile from a CSV
Example from command line:
>> python wave_csv_to_points.py wave_formatted_csv_data.txt my_layer shape_out.shp
>> python wave_csv_to_points.py wave_formatted_csv_data.txt my_layer shape_out.shp
Example two:
>> python wave_csv_to_points.py wave_formatted_csv_data.csv WCVI WCVI_points.shp
>> python wave_csv_to_points.py wave_formatted_csv_data.csv WCVI WCVI_points.shp
"""
# Imports
import csv
from osgeo import ogr
from osgeo import osr
import os
import sys
import shutil
from osgeo import ogr
from osgeo import osr
def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
"""Creates a point shapefile from a wave energy data csv file that is
"""Creates a point shapefile from a wave energy data csv file that is
properly formatted. The point shapefile created is not projected
and uses latitude and longitude for its geometry.
wave_data_csv_uri - a URI to a comma separated file of wave point data
that has been properly formatted (required) Example format:
ID,I,J,LONG,LATI,HSAVG,TPAVG
1,102,370,24.3,54.3,10.2,11.1
2,102,370,24.3,54.3,10.2,11.1
layer_name - a string for the name of the point shapefile
layer (required)
output_uri - a URI for the output path of the point shapefile (required)
return - Nothing"""
# Initiate a dictionary to build up data from the csv file
# Initiate a dictionary to build up data from the csv file
dict_data = {}
# Open the csv file
@ -41,7 +43,7 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
# Get a handle on the csv file by using dictReader which handles each line
# as a dictionary where the column headers are the keys
reader = csv.DictReader(point_file)
# A list of column headers that we want to remain integers all other column
# headers will become floats
int_list = ['ID', 'I', 'J']
@ -49,7 +51,7 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
# Iterate over the file by line
for row in reader:
# For each line's dictionary, iterate over the key-value pairs
for k,v in row.iteritems():
for k,v in row.items():
# If the key represents a value that should be an integer, cast to
# an int, else cast to a float
if k in int_list:
@ -58,7 +60,7 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
row[k] = float(v)
# Build up the new dictionary
dict_data[row['ID']] = row
# If the output_uri exists delete it
if os.path.isfile(output_uri):
os.remove(output_uri)
@ -73,19 +75,19 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
# Set the spatial reference to WGS84 (lat/long)
source_sr = osr.SpatialReference()
source_sr.SetWellKnownGeogCS("WGS84")
# Create the new point shapefile layer
output_layer = output_datasource.CreateLayer(
layer_name, source_sr, ogr.wkbPoint)
# Get the keys of the dictionary which are the 'ID' values
# Get the keys of the dictionary which are the 'ID' values
outer_keys = dict_data.keys()
# Using the list of keys from above, get the first keys sub dictionary and
# get it's keys. These 'inner' keys are the column headers from the file
# and will be added to the point shapefile as fields
field_list = dict_data[outer_keys[0]].keys()
# Create a dictionary to store what variable types the fields are
type_dict = {}
@ -107,14 +109,14 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
field_type = ogr.OFTString
else:
field_type = ogr.OFTReal
# Create a new field in the point shapefile
output_field = ogr.FieldDefn(field, field_type)
output_field = ogr.FieldDefn(field, field_type)
output_layer.CreateField(output_field)
# For each inner dictionary (for each point) create a point and set its
# fields
for point_dict in dict_data.itervalues():
for point_dict in dict_data.values():
# Get latitude / longitude values
latitude = float(point_dict['LATI'])
longitude = float(point_dict['LONG'])
@ -125,12 +127,12 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
# Create a new point feature
output_feature = ogr.Feature(output_layer.GetLayerDefn())
for field_name in point_dict:
field_index = output_feature.GetFieldIndex(field_name)
# Set the value for each field for this particular point
output_feature.SetField(field_index, point_dict[field_name])
# Set geometry and create / set the feature
output_feature.SetGeometryDirectly(geom)
output_layer.CreateFeature(output_feature)
@ -138,12 +140,16 @@ def create_wave_point_ds(wave_data_csv_uri, layer_name, output_uri):
output_layer.SyncToDisk()
# Argument 1 from the command line, the wave energy csv data
wave_data_csv_uri = sys.argv[1]
# Argument 2 from the command line, a string for the layer name
layer_name = sys.argv[2]
# Argument 3 from the command line, the output URI for the point shapefile
out_uri = sys.argv[3]
# Call the function to create our point shapefile from CSV
create_wave_point_ds(wave_data_csv_uri, layer_name, out_uri)
if __name__ == '__main__':
# Argument 1 from the command line, the wave energy csv data
wave_data_csv_uri = sys.argv[1]
# Argument 2 from the command line, a string for the layer name
layer_name = sys.argv[2]
# Argument 3 from the command line, the output URI for the point shapefile
out_uri = sys.argv[3]
# Call the function to create our point shapefile from CSV
create_wave_point_ds(wave_data_csv_uri, layer_name, out_uri)

42
scripts/wave_energy_scripts/wave_watch_data_to_binary.py
View File

@ -1,28 +1,30 @@
"""This Python Script converts and compresses yearly wave watch three text data
to a binary format that's faster to load and cheaper to build in an
installer than the bulky text format
Example from command line:
>> python wave_watch_data_to_binary.py ww3_yearly.txt ww3_binary.bin
"""
import numpy as np
import sys
import struct
import numpy as np
def text_wave_data_to_binary(wave_watch_file_uri, binary_file_uri):
"""Convert and compress the wave watch three data into binary format,
packing in a specific manner such that the InVEST3.0 wave energy
model can properly unpack it
wave_watch_file_uri - a URI to the formatted yearly wave watch three
data (required)
binary_file_uri - a URI to write out the binary file (.bin) (required)
returns - Nothing"""
# Open the wave watch three files
wave_file = open(wave_watch_file_uri,'rU')
# Open the binary output file as writeable
@ -30,32 +32,32 @@ def text_wave_data_to_binary(wave_watch_file_uri, binary_file_uri):
# Initiate an empty list for packing up the wave periods
wave_periods = []
# Initiate an empty list for packing up the wave heights
# Initiate an empty list for packing up the wave heights
wave_heights = []
# Get the periods and heights, removing the newline characters and splitting
# on commas
wave_periods = map(float,wave_file.readline().rstrip('\n').split(','))
wave_heights = map(float,wave_file.readline().rstrip('\n').split(','))
# Pack up the number of wave period and wave height entries into two
# integers. This is used to properly unpack
s=struct.pack('ii',len(wave_periods),len(wave_heights))
bin_file.write(s)
# Pack up the wave period values as float types
s=struct.pack('f'*len(wave_periods), *wave_periods)
bin_file.write(s)
# Pack up the wave height values as float types
s=struct.pack('f'*len(wave_heights), *wave_heights)
bin_file.write(s)
# For the rest of the file
# For the rest of the file
while True:
# Get the next line
line = wave_file.readline()
# Check for the end of the file
if len(line) == 0:
#end of file
@ -75,9 +77,13 @@ def text_wave_data_to_binary(wave_watch_file_uri, binary_file_uri):
s=struct.pack('f'*len(float_list), *float_list)
bin_file.write(s)
# Get the wave watch three uri from the first command line argument
wave_watch_file_uri = sys.argv[1]
# Get the out binary uri from the second command line argument
binary_file_uri = sys.argv[2]
# Call the function to properly convert and compress data
text_wave_data_to_binary(wave_watch_file_uri, binary_file_uri)
if __name__ == '__main__':
# Get the wave watch three uri from the first command line argument
wave_watch_file_uri = sys.argv[1]
# Get the out binary uri from the second command line argument
binary_file_uri = sys.argv[2]
# Call the function to properly convert and compress data
text_wave_data_to_binary(wave_watch_file_uri, binary_file_uri)

22
scripts/wave_energy_scripts/ww3_per_year.py
View File

@ -1,13 +1,15 @@
"""This Python Script converts wave watch 3 data to yearly values
Example from command line:
>> python ww3_per_year.py ww3_formatted.txt ww3_yearly.txt 10
>> python ww3_per_year.py ww3_formatted.txt ww3_yearly.txt 10
"""
import os
import numpy as np
import sys
import numpy as np
def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
"""Divide all the wave watch data values by 'num_years' to get per year data
@ -20,11 +22,11 @@ def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
collected over (required)
returns - Nothing"""
# Get the number of years as a float just to make sure we do floating
# division
# division
num_years = float(num_years)
# Open the wave watch text file
wave_file = open(wave_watch_file_uri)
@ -33,7 +35,7 @@ def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
wave_heights = wave_file.readline()
# Open the output uri as writeable
out_file = open(out_file_uri, 'w')
out_file = open(out_file_uri, 'w')
# Write the wave periods and heights to the output file as we DO NOT want to
# divide the ranges by the number of years
out_file.write(wave_periods)
@ -44,7 +46,7 @@ def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
# Get the next line
line = wave_file.readline()
# Set a blank string to build up altered yearly data line
out_string = ''
out_string = ''
# Check for the end of the file
if len(line) == 0:
#end of file
@ -61,7 +63,7 @@ def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
# Divide all the values in the list by the number of years to get
# per year values
per_yr_vals = values / num_years
# Now that we have a list of floating values we need to build up a
# string to write back to the output file. Start by iterating over
# each per year value in the list
@ -75,11 +77,11 @@ def extrapolate_wave_data(wave_watch_file_uri, out_file_uri, num_years):
# Once the string is completed for the line, remove the trailing
# comma at the end of the line and replace with a newline character
out_string = out_string[0:len(out_string)-1] + '\n'
# Write the line string with the yearly data values to the output
# file
out_file.write(out_string)
# Close the files
wave_file.close()
out_file.close()