|
| 1 | +#!/usr/bin/env bash |
| 2 | +# |
| 3 | +# Finds source assets duplicated across the AssetBundle / ContentDirectory boundary of a |
| 4 | +# hybrid Addressables 4.x build (some groups build to .bundle files, others build through |
| 5 | +# BuildPipeline.BuildContentDirectory / ContentDirectoryGroupSchema). |
| 6 | +# |
| 7 | +# Linux/macOS counterpart of Compare-HybridDuplication.ps1 -- same behavior, same queries. |
| 8 | +# Requires UnityDataTool to be built (see ../../../README.md, "How to Build") and sqlite3 on |
| 9 | +# PATH, exactly like the PowerShell version. |
| 10 | +# |
| 11 | +# See ../SKILL.md for the concept, the query this script runs, and how to read the output. |
| 12 | +# |
| 13 | +# DISCLAIMER: |
| 14 | +# This script is provided "as-is," without any warranty of any kind, express or implied. |
| 15 | +# By using this script, you agree that you understand its purpose and that you use it entirely |
| 16 | +# at your own risk. Always review and test this script in a safe environment before applying it |
| 17 | +# to a production system. |
| 18 | + |
| 19 | +set -euo pipefail |
| 20 | + |
| 21 | +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 22 | + |
| 23 | +project_root="." |
| 24 | +tool_path="" |
| 25 | +build_layout="" |
| 26 | +content_directory="" |
| 27 | +build_history="" |
| 28 | +database="" |
| 29 | +keep_database=0 |
| 30 | +max_rows=50 |
| 31 | + |
| 32 | +usage() { |
| 33 | + cat <<'EOF' |
| 34 | +Usage: compare-hybrid-duplication.sh [options] |
| 35 | +
|
| 36 | + --project-root <path> Root of the Unity project to audit (default: .) |
| 37 | + --tool-path <path> Path to the UnityDataTool executable |
| 38 | + --build-layout <path> Addressables build layout report to read |
| 39 | + (default: Library/com.unity.addressables/buildlayout.json) |
| 40 | + --content-directory <path> Content directory build output folder |
| 41 | + (default: auto-detected under Library/com.unity.addressables/aa) |
| 42 | + --build-history <path> Project's build history folder |
| 43 | + (default: Library/BuildHistory) |
| 44 | + --database <path> Output database path (default: a temp file, deleted afterwards) |
| 45 | + --keep-database Keep the generated database instead of deleting it |
| 46 | + --max-rows <n> Maximum number of duplicate rows to print (default: 50) |
| 47 | + -h, --help Show this help |
| 48 | +EOF |
| 49 | +} |
| 50 | + |
| 51 | +while [[ $# -gt 0 ]]; do |
| 52 | + case "$1" in |
| 53 | + --project-root) project_root="$2"; shift 2 ;; |
| 54 | + --tool-path) tool_path="$2"; shift 2 ;; |
| 55 | + --build-layout) build_layout="$2"; shift 2 ;; |
| 56 | + --content-directory) content_directory="$2"; shift 2 ;; |
| 57 | + --build-history) build_history="$2"; shift 2 ;; |
| 58 | + --database) database="$2"; shift 2 ;; |
| 59 | + --keep-database) keep_database=1; shift ;; |
| 60 | + --max-rows) max_rows="$2"; shift 2 ;; |
| 61 | + -h|--help) usage; exit 0 ;; |
| 62 | + *) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;; |
| 63 | + esac |
| 64 | +done |
| 65 | + |
| 66 | +add_commas() { |
| 67 | + printf "%s" "$1" | sed -E ':a;s/(^[-]?[0-9]+)([0-9]{3})/\1,\2/;ta' |
| 68 | +} |
| 69 | + |
| 70 | +# Resolve the UnityDataTool executable: explicit flag -> UNITYDATATOOL_PATH -> PATH -> this |
| 71 | +# checkout's own build output (only valid while the skill is still inside a UnityDataTools |
| 72 | +# clone; once copied into another project's .claude/skills, pass --tool-path or set |
| 73 | +# UNITYDATATOOL_PATH instead). |
| 74 | +if [[ -z "$tool_path" ]]; then |
| 75 | + if [[ -n "${UNITYDATATOOL_PATH:-}" ]]; then |
| 76 | + tool_path="$UNITYDATATOOL_PATH" |
| 77 | + elif command -v UnityDataTool >/dev/null 2>&1; then |
| 78 | + tool_path="$(command -v UnityDataTool)" |
| 79 | + elif [[ -x "$script_dir/../../../UnityDataTool/bin/Release/net9.0/UnityDataTool" ]]; then |
| 80 | + tool_path="$script_dir/../../../UnityDataTool/bin/Release/net9.0/UnityDataTool" |
| 81 | + fi |
| 82 | +fi |
| 83 | + |
| 84 | +if [[ -z "$tool_path" || ! -x "$tool_path" ]]; then |
| 85 | + echo "Error: UnityDataTool executable not found. Tried: --tool-path, UNITYDATATOOL_PATH, PATH, and this checkout's own build output. Build it with 'dotnet build -c Release' or pass --tool-path explicitly." >&2 |
| 86 | + exit 1 |
| 87 | +fi |
| 88 | + |
| 89 | +if [[ ! -d "$project_root" ]]; then |
| 90 | + echo "Error: Project root '$project_root' not found." >&2 |
| 91 | + exit 1 |
| 92 | +fi |
| 93 | +project_root="$(cd "$project_root" && pwd)" |
| 94 | + |
| 95 | +if [[ -z "$build_layout" ]]; then |
| 96 | + build_layout="$project_root/Library/com.unity.addressables/buildlayout.json" |
| 97 | +fi |
| 98 | +if [[ ! -f "$build_layout" ]]; then |
| 99 | + echo "Error: Addressables build layout report not found at '$build_layout'. Build the project with Addressables first, or pass --build-layout explicitly." >&2 |
| 100 | + exit 1 |
| 101 | +fi |
| 102 | + |
| 103 | +if [[ -z "$content_directory" ]]; then |
| 104 | + candidates=() |
| 105 | + if [[ -d "$project_root/Library/com.unity.addressables/aa" ]]; then |
| 106 | + while IFS= read -r -d '' manifest_hash_file; do |
| 107 | + candidates+=("$(dirname "$manifest_hash_file")") |
| 108 | + done < <(find "$project_root/Library/com.unity.addressables/aa" -name "BuildManifestHash.txt" -print0 2>/dev/null) |
| 109 | + fi |
| 110 | + |
| 111 | + if [[ ${#candidates[@]} -eq 0 ]]; then |
| 112 | + echo "Error: No content directory build output found under Library/com.unity.addressables/aa. Pass --content-directory explicitly, or this project may not have a ContentDirectoryGroupSchema group." >&2 |
| 113 | + exit 1 |
| 114 | + fi |
| 115 | + if [[ ${#candidates[@]} -gt 1 ]]; then |
| 116 | + echo "Error: More than one content directory build output found (multiple platforms built?): ${candidates[*]}. Pass --content-directory to pick one." >&2 |
| 117 | + exit 1 |
| 118 | + fi |
| 119 | + content_directory="${candidates[0]}" |
| 120 | +fi |
| 121 | +if [[ ! -d "$content_directory" ]]; then |
| 122 | + echo "Error: Content directory build output '$content_directory' not found." >&2 |
| 123 | + exit 1 |
| 124 | +fi |
| 125 | + |
| 126 | +if [[ -z "$build_history" ]]; then |
| 127 | + build_history="$project_root/Library/BuildHistory" |
| 128 | +fi |
| 129 | +if [[ ! -d "$build_history" ]]; then |
| 130 | + echo "Error: Build history folder '$build_history' not found. It is required to pair the content directory output with its ContentLayout.json." >&2 |
| 131 | + exit 1 |
| 132 | +fi |
| 133 | + |
| 134 | +delete_database_after=0 |
| 135 | +if [[ -z "$database" ]]; then |
| 136 | + database="$(mktemp -u "${TMPDIR:-/tmp}/hybrid-duplication-XXXXXX.db")" |
| 137 | + if [[ "$keep_database" -eq 0 ]]; then |
| 138 | + delete_database_after=1 |
| 139 | + fi |
| 140 | +fi |
| 141 | +cleanup() { |
| 142 | + if [[ "$delete_database_after" -eq 1 ]]; then |
| 143 | + rm -f "$database" |
| 144 | + fi |
| 145 | +} |
| 146 | +trap cleanup EXIT |
| 147 | + |
| 148 | +echo "Analyzing:" |
| 149 | +echo " Build layout: $build_layout" |
| 150 | +echo " Content directory: $content_directory" |
| 151 | +echo " Build history: $build_history" |
| 152 | +echo "" |
| 153 | + |
| 154 | +"$tool_path" analyze "$build_layout" "$content_directory" --build-history "$build_history" -o "$database" |
| 155 | + |
| 156 | +# Hybrid check: both a bundle-producing group and a content-directory build must be present, |
| 157 | +# otherwise the intersection below is meaningless. |
| 158 | +bundle_count="$(sqlite3 "$database" "SELECT COUNT(*) FROM addressables_build_bundles;")" |
| 159 | +has_layout_table="$(sqlite3 "$database" "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='content_layout_source_assets';")" |
| 160 | +source_asset_count=0 |
| 161 | +if [[ "$has_layout_table" -gt 0 ]]; then |
| 162 | + source_asset_count="$(sqlite3 "$database" "SELECT COUNT(*) FROM content_layout_source_assets;")" |
| 163 | +fi |
| 164 | + |
| 165 | +if [[ "$bundle_count" -eq 0 || "$source_asset_count" -eq 0 ]]; then |
| 166 | + echo "This is not a hybrid build:" |
| 167 | + echo " AssetBundle groups in the build layout: $bundle_count" |
| 168 | + echo " Source assets in the content directory: $source_asset_count" |
| 169 | + echo "" |
| 170 | + echo "Both must be non-zero to audit cross-boundary duplication." |
| 171 | + exit 2 |
| 172 | +fi |
| 173 | + |
| 174 | +# The one query that answers "which source assets were built into both forms": bundle-side |
| 175 | +# asset paths (explicit + implicit) intersected with the content directory's source assets. |
| 176 | +# Matching by asset_path, not by object CRC, is what makes this catch cases a plain CRC diff |
| 177 | +# (e.g. view_potential_duplicates, see Documentation/analyzer.md) would miss -- the same |
| 178 | +# source asset can be built with a different variant set on each side, so the two copies do |
| 179 | +# not have the same CRC despite being duplicates of the same asset. |
| 180 | +duplicates_query=" |
| 181 | +WITH bundle_assets AS ( |
| 182 | + SELECT asset_path, serialized_size + streamed_size AS bytes |
| 183 | + FROM addressables_build_explicit_assets |
| 184 | + UNION ALL |
| 185 | + SELECT asset_path, serialized_size + streamed_size AS bytes |
| 186 | + FROM addressables_build_data_from_other_assets |
| 187 | +), |
| 188 | +content_dir_assets AS ( |
| 189 | + SELECT DISTINCT asset_path FROM content_layout_source_assets |
| 190 | +) |
| 191 | +SELECT b.asset_path, SUM(b.bytes) AS bundle_bytes, COUNT(*) AS instances |
| 192 | +FROM bundle_assets b |
| 193 | +WHERE b.asset_path IN (SELECT asset_path FROM content_dir_assets) |
| 194 | +GROUP BY b.asset_path |
| 195 | +ORDER BY bundle_bytes DESC |
| 196 | +LIMIT $max_rows; |
| 197 | +" |
| 198 | + |
| 199 | +totals_query=" |
| 200 | +WITH bundle_assets AS ( |
| 201 | + SELECT serialized_size + streamed_size AS bytes FROM addressables_build_explicit_assets |
| 202 | + UNION ALL |
| 203 | + SELECT serialized_size + streamed_size AS bytes FROM addressables_build_data_from_other_assets |
| 204 | +) |
| 205 | +SELECT SUM(bytes) FROM bundle_assets; |
| 206 | +" |
| 207 | + |
| 208 | +content_dir_total_query="SELECT SUM(size) FROM content_layout_binary_artifacts WHERE category != 'manifest';" |
| 209 | + |
| 210 | +duplicate_bytes_query=" |
| 211 | +WITH bundle_assets AS ( |
| 212 | + SELECT asset_path, serialized_size + streamed_size AS bytes FROM addressables_build_explicit_assets |
| 213 | + UNION ALL |
| 214 | + SELECT asset_path, serialized_size + streamed_size AS bytes FROM addressables_build_data_from_other_assets |
| 215 | +) |
| 216 | +SELECT COALESCE(SUM(bytes), 0) |
| 217 | +FROM bundle_assets |
| 218 | +WHERE asset_path IN (SELECT DISTINCT asset_path FROM content_layout_source_assets); |
| 219 | +" |
| 220 | + |
| 221 | +bundle_total_bytes="$(sqlite3 "$database" "$totals_query")" |
| 222 | +content_dir_total_bytes="$(sqlite3 "$database" "$content_dir_total_query")" |
| 223 | +duplicate_bytes="$(sqlite3 "$database" "$duplicate_bytes_query")" |
| 224 | + |
| 225 | +declared_duplicate_count="$(grep -o '"DuplicatedAssetCount"[[:space:]]*:[[:space:]]*-\{0,1\}[0-9]*' "$build_layout" | grep -o -- '-\{0,1\}[0-9]*$' || true)" |
| 226 | + |
| 227 | +echo "=== Cross-boundary duplicates (built into both a bundle and the content directory) ===" |
| 228 | +sqlite3 "$database" ".mode column" ".headers on" "$duplicates_query" |
| 229 | +echo "" |
| 230 | + |
| 231 | +echo "=== Summary ===" |
| 232 | +printf "Bundle-side asset payload: %15s bytes\n" "$(add_commas "$bundle_total_bytes")" |
| 233 | +printf "Content directory payload: %15s bytes\n" "$(add_commas "$content_dir_total_bytes")" |
| 234 | +printf "Cross-boundary duplicated: %15s bytes\n" "$(add_commas "$duplicate_bytes")" |
| 235 | +if [[ "$bundle_total_bytes" -gt 0 ]]; then |
| 236 | + pct="$(awk -v d="$duplicate_bytes" -v t="$bundle_total_bytes" 'BEGIN { printf "%.1f", (100.0 * d / t) }')" |
| 237 | + echo " = ${pct}% of the bundle-side payload" |
| 238 | +fi |
| 239 | +if [[ -n "$declared_duplicate_count" ]]; then |
| 240 | + echo "" |
| 241 | + echo "Addressables' own DuplicatedAssetCount for this build: $declared_duplicate_count" |
| 242 | + echo "(That count only scans AssetBundle-to-AssetBundle duplication -- it cannot see a copy that lives in the content directory, so it will read low or zero even when the total above is not.)" |
| 243 | +fi |
| 244 | + |
| 245 | +if [[ "$delete_database_after" -eq 0 ]]; then |
| 246 | + echo "" |
| 247 | + echo "Database kept at: $database" |
| 248 | +fi |
0 commit comments