Move all the topo stac creation into cogify

packages/cogify/src/cogify/__test__/covering.test.ts

@@ -4,7 +4,7 @@ import { describe, it } from 'node:test';
 import { GoogleTms, QuadKey } from '@basemaps/geo';
 
 import { gsdToMeter } from '../cli/cli.cover.js';
-import { addChildren, addSurrounding } from '../covering.js';
+import { addChildren, addSurrounding } from '../covering/covering.js';
 
 describe('getChildren', () => {
   it('should get children', () => {

packages/cogify/src/cogify/__test__/extract.test.ts

@@ -0,0 +1,37 @@
+import { strictEqual, throws } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { extractMapCodeAndVersion } from '../topo/extract.js';
+
+describe('extractMapCodeAndVersion', () => {
+  const FakeDomain = 's3://topographic/fake-domain';
+  const validFiles = [
+    { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    {
+      input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' },
+    },
+    {
+      input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' },
+    },
+  ];
+  const invalidFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`];
+
+  it('should parse the correct MapSheet Names', () => {
+    for (const file of validFiles) {
+      const output = extractMapCodeAndVersion(file.input);
+      strictEqual(output.mapCode, file.expected.mapCode, 'Map code does not match');
+      strictEqual(output.version, file.expected.version, 'Version does not match');
+    }
+  });
+
+  it('should not able to parse a version from file', () => {
+    for (const file of invalidFiles) {
+      throws(() => extractMapCodeAndVersion(file), new Error('Version not found in the file name'));
+    }
+  });
+});

packages/cogify/src/cogify/cli/cli.cog.ts

@@ -11,18 +11,18 @@ import path from 'path';
 import { StacAsset, StacCollection } from 'stac-ts';
 import { pathToFileURL } from 'url';
 
-import { CutlineOptimizer } from '../../cutline.js';
 import { SourceDownloader } from '../../download.js';
 import { HashTransform } from '../../hash.stream.js';
 import { getLogger, logArguments } from '../../log.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
 import {
   gdalBuildCog,
   gdalBuildTopoRasterCommands,
   gdalBuildVrt,
   gdalBuildVrtWarp,
   gdalCreate,
-} from '../gdal.command.js';
-import { GdalRunner } from '../gdal.runner.js';
+} from '../gdal/gdal.command.js';
+import { GdalRunner } from '../gdal/gdal.runner.js';
 import { Url, UrlArrayJsonFile } from '../parsers.js';
 import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
 

packages/cogify/src/cogify/cli/cli.cover.ts

@@ -7,10 +7,10 @@ import { Metrics } from '@linzjs/metrics';
 import { command, flag, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
 
 import { isArgo } from '../../argo.js';
-import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { Presets } from '../../preset.js';
-import { createTileCover, TileCoverContext } from '../../tile.cover.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import { createTileCover, TileCoverContext } from '../covering/tile.cover.js';
 import { RgbaType, Url, UrlFolder } from '../parsers.js';
 import { createFileStats } from '../stac.js';
 

packages/cogify/src/cogify/cli/cli.topo.ts

@@ -0,0 +1,207 @@
+import { loadTiffsFromPaths } from '@basemaps/config-loader/build/json/tiff.config.js';
+import { Bounds, Epsg, Nztm2000Tms, TileMatrixSets } from '@basemaps/geo';
+import { fsa, LogType } from '@basemaps/shared';
+import { CliInfo } from '@basemaps/shared/build/cli/info.js';
+import { boolean, command, flag, option, string } from 'cmd-ts';
+import pLimit from 'p-limit';
+
+import { isArgo } from '../../argo.js';
+import { UrlFolder } from '../../cogify/parsers.js';
+import { getLogger, logArguments } from '../../log.js';
+import { TopoStacItem } from '../stac.js';
+import { groupTiffsByDirectory, mapEpsgToSlug } from '../topo/mapper.js';
+import { createStacCollection, createStacItems, writeStacFiles } from '../topo/stac.creation.js';
+import { brokenTiffs } from '../topo/types.js';
+
+const Q = pLimit(10);
+
+/**
+ * List all the tiffs in a directory for topographic maps and create cogs for each.
+ *
+ * @param source: Location of the source files
+ * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/
+ *
+ * @param target: Location of the target path
+ */
+export const TopoStacCreationCommand = command({
+  name: 'cogify-topo-stac',
+  version: CliInfo.version,
+  description: 'List input topographic files, create StacItems, and generate tiles for grouping.',
+  args: {
+    ...logArguments,
+    title: option({
+      type: string,
+      long: 'title',
+      description: 'Imported imagery title',
+    }),
+    source: option({
+      type: UrlFolder,
+      long: 'source',
+      description: 'Location of the source files',
+    }),
+    target: option({
+      type: UrlFolder,
+      long: 'target',
+      description: 'Target location for the output files',
+    }),
+    scale: option({
+      type: string,
+      long: 'scale',
+      description: 'topo25, topo50, or topo250',
+    }),
+    resolution: option({
+      type: string,
+      long: 'resolution',
+      description: 'e.g. gridless_600dpi',
+    }),
+    latestOnly: flag({
+      type: boolean,
+      defaultValue: () => false,
+      long: 'latest-only',
+      description: 'Only process the latest version of each map sheet',
+      defaultValueIsSerializable: true,
+    }),
+    forceOutput: flag({
+      type: boolean,
+      defaultValue: () => false,
+      long: 'force-output',
+      defaultValueIsSerializable: true,
+    }),
+  },
+  async handler(args) {
+    const logger = getLogger(this, args);
+    const startTime = performance.now();
+    logger.info('ListJobs:Start');
+
+    const { epsgDirectoryPaths, stacItemPaths } = await loadTiffsToCreateStacs(
+      args.latestOnly,
+      args.source,
+      args.target,
+      args.title,
+      args.scale,
+      args.resolution,
+      args.forceOutput,
+      logger,
+    );
+
+    if (epsgDirectoryPaths.length === 0 || stacItemPaths.length === 0) throw new Error('No Stac items created');
+
+    // write stac items into an JSON array
+    if (args.forceOutput || isArgo()) {
+      const targetURL = isArgo() ? new URL('/tmp/topo-stac-creation/') : args.target;
+
+      // for create-config: we need to tell create-config to create a bundled config for each epsg folder (latest only).
+      // workflow: will loop 'targets.json' and create a node for each path where each node's job is to create a bundled config.
+      await fsa.write(new URL('targets.json', targetURL), JSON.stringify(epsgDirectoryPaths, null, 2));
+
+      // tiles.json makes the tiff files
+      await fsa.write(new URL('tiles.json', targetURL), JSON.stringify(stacItemPaths, null, 2));
+      await fsa.write(new URL('brokenTiffs.json', targetURL), JSON.stringify(brokenTiffs, null, 2));
+    }
+
+    logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done');
+  },
+});
+
+/**
+ * @param source: Source directory URL from which to load tiff files
+ * @example TODO
+ *
+ * @param target: Destination directory URL into which to save the STAC collection and item JSON files
+ * @example TODO
+ *
+ * @param title: The title of the collection
+ * @example "New Zealand Topo50 Map Series (Gridless)"
+ *
+ * @returns an array of StacItem objects
+ */
+async function loadTiffsToCreateStacs(
+  latestOnly: boolean,
+  source: URL,
+  target: URL,
+  title: string,
+  scale: string,
+  resolution: string,
+  forceOutput: boolean,
+  logger?: LogType,
+): Promise<{ epsgDirectoryPaths: { epsg: string; url: URL }[]; stacItemPaths: { path: URL }[] }> {
+  logger?.info({ source }, 'LoadTiffs:Start');
+  // extract all file paths from the source directory and convert them into URL objects
+  const fileURLs = await fsa.toArray(fsa.list(source));
+  // process all of the URL objects into Tiff objects
+  const tiffs = await loadTiffsFromPaths(fileURLs, Q);
+  logger?.info({ numTiffs: tiffs.length }, 'LoadTiffs:End');
+
+  // group all of the Tiff objects by epsg and map code
+  logger?.info('GroupTiffs:Start');
+  const itemsByDir = groupTiffsByDirectory(tiffs, logger);
+  const itemsByDirPath = new URL('itemsByDirectory.json', target);
+  await fsa.write(itemsByDirPath, JSON.stringify(itemsByDir, null, 2));
+  logger?.info('GroupTiffs:End');
+
+  const epsgDirectoryPaths: { epsg: string; url: URL }[] = [];
+  const stacItemPaths = [];
+
+  // create and write stac items and collections
+  for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) {
+    const allTargetURL = new URL(`${scale}/${resolution}/${epsg}/`, target);
+    const latestTargetURL = new URL(`${scale}_latest/${resolution}/${epsg}/`, target);
+
+    const allBounds: Bounds[] = [];
+    const allStacItems: TopoStacItem[] = [];
+
+    const latestBounds: Bounds[] = [];
+    const latestStacItems: TopoStacItem[] = [];
+
+    // parse epsg
+    const epsgCode = Epsg.parse(epsg);
+    if (epsgCode == null) throw new Error(`Failed to parse epsg '${epsg}'`);
+
+    // convert epsg to tile matrix
+    const tileMatrix = TileMatrixSets.tryGet(epsgCode) ?? Nztm2000Tms; // TODO: support other tile matrices
+    if (tileMatrix == null) throw new Error(`Failed to convert epsg code '${epsgCode.code}' to a tile matrix`);
+
+    // create stac items
+    logger?.info({ epsg }, 'CreateStacItems:Start');
+    for (const [mapCode, items] of itemsByMapCode.entries()) {
+      // get latest item
+      const latest = itemsByDir.latest.get(epsg).get(mapCode);
+
+      // create stac items
+      const stacItems = createStacItems(allTargetURL, tileMatrix, items, latest, logger);
+
+      allBounds.push(...items.map((item) => item.bounds));
+      allStacItems.push(...stacItems.all);
+
+      latestBounds.push(latest.bounds);
+      latestStacItems.push(stacItems.latest);
+    }
+
+    // convert epsg to slug
+    const epsgSlug = mapEpsgToSlug(epsgCode.code);
+    if (epsgSlug == null) throw new Error(`Failed to map epsg code '${epsgCode.code}' to a slug`);
+
+    const linzSlug = `${scale}-${epsgSlug}`;
+
+    // create collections
+    const collection = createStacCollection(title, linzSlug, Bounds.union(allBounds), allStacItems, logger);
+    const latestCollection = createStacCollection(title, linzSlug, Bounds.union(latestBounds), latestStacItems, logger);
+    logger?.info({ epsg }, 'CreateStacItems:End');
+
+    if (forceOutput || isArgo()) {
+      epsgDirectoryPaths.push({ epsg, url: latestTargetURL });
+
+      // write stac items and collections
+      logger?.info({ epsg }, 'WriteStacFiles:Start');
+      if (!latestOnly) {
+        const allPaths = await writeStacFiles(allTargetURL, allStacItems, collection, logger);
+        stacItemPaths.push(...allPaths.itemPaths);
+      }
+      const latestPaths = await writeStacFiles(latestTargetURL, latestStacItems, latestCollection, logger);
+      stacItemPaths.push(...latestPaths.itemPaths);
+      logger?.info({ epsg }, 'WriteStacFiles:End');
+    }
+  }
+
+  return { epsgDirectoryPaths, stacItemPaths };
+}

packages/cogify/src/cogify/covering.ts → packages/cogify/src/cogify/covering/covering.ts

@@ -3,7 +3,7 @@ import { LogType } from '@basemaps/shared';
 import { Area, intersection, MultiPolygon } from '@linzjs/geojson';
 import { Metrics } from '@linzjs/metrics';
 
-import { CutlineOptimizer } from '../cutline.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
 
 /**
  * Tile offsets for surrounding tiles

packages/cogify/src/cutline.ts → packages/cogify/src/cogify/covering/cutline.ts

@@ -10,7 +10,7 @@ import {
   toFeatureMultiPolygon,
 } from '@linzjs/geojson';
 
-import { CogifyLinkCutline } from './cogify/stac.js';
+import { CogifyLinkCutline } from '../stac.js';
 
 export async function loadCutline(path: URL): Promise<{ polygon: MultiPolygon; projection: EpsgCode }> {
   const buf = await fsa.read(path);

packages/cogify/src/tile.cover.ts → packages/cogify/src/cogify/covering/tile.cover.ts

@@ -7,16 +7,10 @@ import { intersection, MultiPolygon, toFeatureCollection, union } from '@linzjs/
 import { Metrics } from '@linzjs/metrics';
 import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js';
 
-import { createCovering } from './cogify/covering.js';
-import {
-  CogifyLinkCutline,
-  CogifyLinkSource,
-  CogifyStacCollection,
-  CogifyStacItem,
-  createFileStats,
-} from './cogify/stac.js';
+import { Presets } from '../../preset.js';
+import { CogifyLinkCutline, CogifyLinkSource, CogifyStacCollection, CogifyStacItem, createFileStats } from '../stac.js';
+import { createCovering } from './covering.js';
 import { CutlineOptimizer } from './cutline.js';
-import { Presets } from './preset.js';
 
 export interface TileCoverContext {
   /** Unique id for the covering */

packages/cogify/src/cogify/gdal.command.ts → packages/cogify/src/cogify/gdal/gdal.command.ts

@@ -2,9 +2,9 @@ import { Rgba } from '@basemaps/config';
 import { Epsg, EpsgCode, TileMatrixSets } from '@basemaps/geo';
 import { urlToString } from '@basemaps/shared';
 
-import { Presets } from '../preset.js';
+import { Presets } from '../../preset.js';
+import { CogifyCreationOptions } from '../stac.js';
 import { GdalCommand } from './gdal.runner.js';
-import { CogifyCreationOptions } from './stac.js';
 
 const isPowerOfTwo = (x: number): boolean => (x & (x - 1)) === 0;
 const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; // 1.7 pixels to trim from the right side of the topo raster imagery