Skip to content

Commit

Permalink
Merge pull request #562 from NASA-AMMOS/jr-561
Browse files Browse the repository at this point in the history
Improvements to bulk_tiles and added quantize_colormap #561
  • Loading branch information
jtroberts authored Jun 20, 2024
2 parents 66b0e55 + fc918f1 commit d2d049f
Show file tree
Hide file tree
Showing 4 changed files with 206 additions and 9 deletions.
1 change: 1 addition & 0 deletions auxiliary/bulk_tiles/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ optional arguments:
-h, --help show this help message and exit
-c COLORMAP_DIR, --colormap_dir COLORMAP_DIR
Directory containing colormaps
-d, --discrete Use discrete color values
-i INPUT_DIR, --input_dir INPUT_DIR
Directory containing raw input TIFF files
-j JSON_CONFIG, --json_config JSON_CONFIG
Expand Down
34 changes: 25 additions & 9 deletions auxiliary/bulk_tiles/bulk_tiles.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3

"""
This script bulk processes a directory of TIFs into TMS tiles
Expand All @@ -15,7 +15,7 @@
from lxml import etree


def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''):
def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, discrete, prefix=''):
output_files = []
if os.path.isdir(input_dir):
input_files = sorted(glob.glob(input_dir + '/' + prefix + '*.tif', recursive=True))
Expand All @@ -25,7 +25,12 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''):
colormap_files = sorted(glob.glob(colormap_dir+'/*.txt', recursive=True))
colormap_dict = {}
for colormap_file in colormap_files:
colormap_key = os.path.basename(colormap_file).split('_')[1].split('.')[0]
if 'units_' in colormap_file:
continue
elif 'colormap_' in colormap_file:
colormap_key = os.path.basename(colormap_file).split('_')[1].split('.')[0]
else:
colormap_key = os.path.basename(colormap_file).split('.')[0]
colormap_dict[colormap_key] = colormap_file
else:
print('Warning: ' + colormap_dir + ' directory does not exist')
Expand All @@ -39,7 +44,10 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''):
# Figure out the colormap map to use for the file
colormap = None
for key in colormap_dict:
if key in input_file:
if os.path.basename(input_file).split('.')[0] == key:
colormap = colormap_dict[key]
break
elif key in input_file:
colormap = colormap_dict[key]
if colormap is None:
print('Warning: Skipping...no colormap found for ' + input_file)
Expand All @@ -50,7 +58,9 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''):
# Colorize the TIFF file with its respective colormap
rastertolegend = str(Path(os.path.dirname(os.path.realpath(__file__)) + '/../rastertolegend/rastertolegend.py')
.absolute())
color_tiff = ['python', rastertolegend, input_file, colormap, '-discrete']
color_tiff = ['python3', rastertolegend, input_file, colormap]
if discrete:
color_tiff.append('-discrete')
print("Running:", " ".join(color_tiff))
process = subprocess.Popen(color_tiff, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait()
Expand All @@ -66,7 +76,7 @@ def process_tiffs(input_dir, process_dir, colormap_dir, legends_dir, prefix=''):
print('Moving ' + color_output + ' to ' + legends_dir + '/' + legend_file)
shutil.move(color_output, legends_dir + '/' + legend_file)
else:
output_file = process_dir + '/' + str(Path(color_output).name)
output_file = process_dir + '/' + str(Path(input_file).name)
print('Moving ' + color_output + ' to ' + output_file)
shutil.move(color_output, output_file)
output_files.append(output_file)
Expand All @@ -88,7 +98,7 @@ def create_tiles(input_files, output_dir):
print('Output directory: ' + output_subdir)
rasterstotiles = str(Path(os.path.dirname(os.path.realpath(__file__)) + '/../rasterstotiles/rasterstotiles.py')
.absolute())
tile_tiff = ['python', rasterstotiles, '-o', output_subdir, tiff]
tile_tiff = ['python3', rasterstotiles, '-o', output_subdir, tiff]
print("Running:", " ".join(tile_tiff))
process = subprocess.Popen(tile_tiff, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait()
Expand Down Expand Up @@ -137,7 +147,7 @@ def create_configs(output_dirs, json_config, prefix):
sublayer['togglesWithHeader'] = False
sublayer['minZoom'] = int(minZoom)
sublayer['maxNativeZoom'] = int(maxNativeZoom)
sublayer['maxZoom'] = 10
sublayer['maxZoom'] = 16
sublayer['boundingBox'] = [minx, miny, maxx, maxy]

sublayers.append(sublayer)
Expand All @@ -155,6 +165,12 @@ def create_configs(output_dirs, json_config, prefix):
dest='colormap_dir',
help='Directory containing colormaps',
action='store')
parser.add_argument(
'-d',
'--discrete',
dest='discrete',
help='Use discrete color values',
action='store_true')
parser.add_argument(
'-i',
'--input_dir',
Expand Down Expand Up @@ -199,7 +215,7 @@ def create_configs(output_dirs, json_config, prefix):
args = parser.parse_args()

output_dirs = glob.glob(args.output_dir + '/*', recursive=True)
output_tiffs = process_tiffs(args.input_dir, args.process_dir, args.colormap_dir, args.legends_dir, args.prefix)
output_tiffs = process_tiffs(args.input_dir, args.process_dir, args.colormap_dir, args.legends_dir, args.discrete, args.prefix)
output_dirs = create_tiles(output_tiffs, args.output_dir)

# Generate JSON layer configurations if specified
Expand Down
20 changes: 20 additions & 0 deletions auxiliary/quantize_colormap/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# quantize_colormap.py

```shell
usage: quantize_colormap.py [-h] [-c COLORMAP] [-i INPUT] [-n NODATA] [-o OUTPUT_DIR] [-0] [-m]

Generate quantized MPL colormaps from tiff files.

options:
-h, --help show this help message and exit
-c COLORMAP, --colormap COLORMAP
matplotlib colormap to use
-i INPUT, --input INPUT
TIFF file or directory containing raw input TIFF files
-n NODATA, --nodata NODATA
nodata value
-o OUTPUT_DIR, --output_dir OUTPUT_DIR
Directory containing output colormaps
-0, --ignore_zeroes Ignore zero and under values
-m, --ignore_min Ignore min value
```
160 changes: 160 additions & 0 deletions auxiliary/quantize_colormap/quantize_colormap.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
#!/usr/bin/env python3

import os
import sys
import argparse
import glob
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from osgeo import gdal

# https://matplotlib.org/stable/gallery/color/colormap_reference.html
cmaps = [('Perceptually Uniform Sequential', [
'viridis', 'plasma', 'inferno', 'magma', 'cividis']),
('Sequential', [
'Greys', 'Purples', 'Blues', 'Greens', 'Oranges', 'Reds',
'YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu',
'GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn']),
('Sequential (2)', [
'binary', 'gist_yarg', 'gist_gray', 'gray', 'bone', 'pink',
'spring', 'summer', 'autumn', 'winter', 'cool', 'Wistia',
'hot', 'afmhot', 'gist_heat', 'copper']),
('Diverging', [
'PiYG', 'PRGn', 'BrBG', 'PuOr', 'RdGy', 'RdBu',
'RdYlBu', 'RdYlGn', 'Spectral', 'coolwarm', 'bwr', 'seismic']),
('Cyclic', ['twilight', 'twilight_shifted', 'hsv']),
('Qualitative', [
'Pastel1', 'Pastel2', 'Paired', 'Accent',
'Dark2', 'Set1', 'Set2', 'Set3',
'tab10', 'tab20', 'tab20b', 'tab20c']),
('Miscellaneous', [
'flag', 'prism', 'ocean', 'gist_earth', 'terrain', 'gist_stern',
'gnuplot', 'gnuplot2', 'CMRmap', 'cubehelix', 'brg',
'gist_rainbow', 'rainbow', 'jet', 'turbo', 'nipy_spectral',
'gist_ncar'])]


class MplColorHelper:

def __init__(self, cmap_name, start_val, stop_val):
self.cmap_name = cmap_name
self.cmap = plt.get_cmap(cmap_name)
self.norm = mpl.colors.Normalize(vmin=start_val, vmax=stop_val)
self.scalarMap = cm.ScalarMappable(norm=self.norm, cmap=self.cmap)

def get_rgb(self, val):
return self.scalarMap.to_rgba(val)


def quantize_colormap_from_tiff(tiff, output_dir, colormap, nodata, ignore_zeroes, ignore_min):
print('Processing', tiff)
output_file = output_dir + '/' + os.path.basename(tiff).split('.')[0] + '.txt'

# open the dataset and retrieve raster data as an array
dataset = gdal.Open(tiff)
array = dataset.ReadAsArray()

# get min max values
min = np.min(array)
max = np.max(array)

# remove 0 and negative values
if ignore_zeroes:
array = array[array > 0]
# remove min values
if ignore_min:
array = array[array != min]

# use the numpy percentile function to calculate quantile thresholds
print('Calculating quantiles...')
print(max, min)
percentiles = []
for i in np.arange(100, 0, -10).tolist():
percentiles.append(np.percentile(array, i))
percentiles.append(np.min(array))
print(percentiles)

print('Generating', output_file)
out = open(output_file, 'w+')
mch = MplColorHelper(colormap, 0, len(percentiles))
for i in range(0, len(percentiles)):
val = percentiles[i]
rgba = (mch.scalarMap.to_rgba(len(percentiles)-1-i))
r = round(rgba[0] * 255)
g = round(rgba[1] * 255)
b = round(rgba[2] * 255)
a = round(rgba[3] * 255)
out_str = ' '.join([str(round(val)), str(r), str(g), str(b), str(a)])
out.write(out_str + '\n')

if not str(nodata) == 'nv':
if float(nodata) <= min:
out.write(f'{str(nodata)} 0 0 0 0\n')
else:
out.write(f'nv 0 0 0 0\n')
out.seek(0)
print(out.read())
out.close()


parser = argparse.ArgumentParser(description='Generate quantized MPL colormaps from tiff files.')
parser.add_argument(
'-c',
'--colormap',
default='viridis',
dest='colormap',
help='matplotlib colormap to use',
action='store')
parser.add_argument(
'-i',
'--input',
default='Raw/',
dest='input',
help='TIFF file or directory containing raw input TIFF files',
action='store')
parser.add_argument(
'-n',
'--nodata',
dest='nodata',
default='nv',
help='nodata value',
action='store')
parser.add_argument(
'-o',
'--output_dir',
default='Colormaps/',
dest='output_dir',
help='Directory containing output colormaps',
action='store')
parser.add_argument(
'-0',
'--ignore_zeroes',
default=False,
dest='ignore_zeroes',
help='Ignore zero and under values',
action='store_true')
parser.add_argument(
'-m',
'--ignore_min',
default=False,
dest='ignore_min',
help='Ignore min value',
action='store_true')

args = parser.parse_args()

if not os.path.isdir(args.output_dir):
print(args.output_dir, 'is not a directory.')
sys.exit()

if os.path.isfile(args.input):
quantize_colormap_from_tiff(args.input, args.output_dir, args.colormap, args.nodata, args.ignore_zeroes, args.ignore_min)
else:
print('Searching', args.input)
input_files = glob.glob(args.input + '*.tif*', recursive=True)
for input_file in input_files:
quantize_colormap_from_tiff(input_file, args.output_dir, args.colormap, args.nodata, args.ignore_zeroes, args.ignore_min)

sys.exit()

0 comments on commit d2d049f

Please sign in to comment.