Progressive Import

[PurpleBooth]

I run this tool in a Kubernetes cluster, and as such it can be restarted at any point if there is resource contestation or whatever. To help with that I made a script that basically does a progressive import across all the profiles. It's the entrypoint to a dockerfile.

I don't really expect to see this as a feature, but sometimes these sorts of things can give a bit of neat insight into how people are using the tool.

• DATA_DIR is a writeable, persistant bit of storage
• CONFIG_DIR is a readonly storage with the contents from .audible
• OUTPUT_DIR is the destination directory to save audiobooks to in a Author/Title ASIN format
• TEMP_DIR is an optional and temporary storage location

Not sometimes the file naming is a bit screwy and I fallback to the original name of the m4b for author and title, but I have another tool that fixes up my audiobook library so that's not a problem for me.

#!/usr/bin/env bash

set -euo pipefail

if [ "$#" -ne 0 ]; then
	exec "$@"
fi

cat <<EOF
          .   .   .             .
,-. . . ,-| . |-. |  ,-.    ,-. |  .
,-| | | | | | | | |  |-' -- |   |  |
\`-^ \`-^ \`-^ ' ^-' \`' \`-'    \`-' \`' '


EOF

echo "▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓"

export NORMAL_TIMEOUT="${NORMAL_TIMEOUT:-2400}"
export COVER_SIZE_PREFERENCE="${COVER_SIZE_PREFERENCE:-1215}"
export DATA_DIR="${DATA_DIR:-/data}"
export OUTPUT_DIR="${OUTPUT_DIR:-/output}"
export CONFIG_DIR="${CONFIG_DIR:-/config}"
export AUDIBLE_PLUGIN_DIR="/src/plugin_cmds"

TEMP_DIR="${TEMP_DIR:-$(mktemp -d)}"
export TEMP_DIR
export DOWNLOAD_DIR="${DOWNLOAD_DIR:-"$TEMP_DIR/downloads"}"
export AUDIBLE_CONFIG_DIR="$DATA_DIR"

mkdir -p "$DOWNLOAD_DIR" "$DATA_DIR" "$OUTPUT_DIR" "$AUDIBLE_CONFIG_DIR"
rm -rf "${DOWNLOAD_DIR:?}/"*

export HOME="$DATA_DIR"
cd "$DATA_DIR"

if [ ! -f "$AUDIBLE_CONFIG_DIR/config.toml" ] || [[ -L "$AUDIBLE_CONFIG_DIR/config.toml" ]]; then
	cp -vrL "$CONFIG_DIR"/*{.toml,.json} "$AUDIBLE_CONFIG_DIR"
fi

echo Will download from profiles
audible manage profile list
echo "▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓"

mapfile -t PROFILES < <(audible manage profile list | tail -n+4 | head -n-1 | cut -d'|' -f3 | shuf)
for PROFILE in "${PROFILES[@]}"; do
	echo "██████████████████████████████████████████████████████████████████████████████████"
	PROFILE="$(echo "$PROFILE" | tr -d '[:space:]')"
	touch "$DATA_DIR/$PROFILE-library"
	export PROFILE
	audible -P "$PROFILE" activation-bytes >/dev/null
	audible -P "$PROFILE" library export --format tsv --output "$TEMP_DIR/raw-list" --timeout "$NORMAL_TIMEOUT"
	{
		cut -d'	' -f1 | tail -n+2
	} >"$TEMP_DIR/library-list.new" <"$TEMP_DIR/raw-list"
	comm -13 <(sort "$DATA_DIR/$PROFILE-library" | uniq || echo) <(sort "$TEMP_DIR/library-list.new" | uniq) | shuf >"$TEMP_DIR/worklist"

	WORKLIST_SIZE="$(wc -l <"$TEMP_DIR/worklist")"
	LIBRARY_SIZE="$(wc -l <"$TEMP_DIR/library-list.new")"
	echo "Queue now $WORKLIST_SIZE of $PROFILE library size $LIBRARY_SIZE"
	mapfile -t WORK_ITEMS <"$TEMP_DIR/worklist"

	for WORK_ITEM in "${WORK_ITEMS[@]}"; do
		if [ -z "$WORK_ITEM" ]; then
			continue
		fi

		echo "▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒"

		DETAILS="$(grep "^$WORK_ITEM	" "$TEMP_DIR/raw-list")"
		TITLE="$(echo "$DETAILS" | cut -d'	' -f2 | sed -e 's/[^A-Za-z0-9._ -]/_/g')"
		SUBTITLE="$(echo "$DETAILS" | cut -d'	' -f3 | sed -e 's/[^A-Za-z0-9._ -]/_/g')"
		AUTHOR="$(echo "$DETAILS" | cut -d'	' -f4 | sed -e 's/[^A-Za-z0-9._ -]/_/g')"

		echo "Downloading $WORK_ITEM - ${TITLE:-Unknown} by ${AUTHOR:-Unknown}"
		audible -P "$PROFILE" download \
			--filename-mode asin_ascii \
			--asin "$WORK_ITEM" \
			--aax-fallback \
			--pdf \
			--cover --cover-size "$COVER_SIZE_PREFERENCE" \
			--timeout "$NORMAL_TIMEOUT" \
			--output-dir "$DOWNLOAD_DIR" \
			--chapter \
			--annotation \
			--no-confirm \
			--overwrite

		echo "Removing Encryption $WORK_ITEM - ${TITLE:-Unknown} by ${AUTHOR:-Unknown}"
		if compgen -G "$DOWNLOAD_DIR/*.aax" >/dev/null; then
			audible -P "$PROFILE" remove-encryption --input "$DOWNLOAD_DIR/"*".aax" --rebuild-chapters || audible -P "$PROFILE" remove-encryption --input "$DOWNLOAD_DIR/"*".aax"
		else
			audible -P "$PROFILE" remove-encryption --input "$DOWNLOAD_DIR/"*".aaxc" --rebuild-chapters || audible -P "$PROFILE" remove-encryption --input "$DOWNLOAD_DIR/"*".aaxc"
		fi

		echo "Moving files $WORK_ITEM - ${TITLE:-Unknown} by ${AUTHOR:-Unknown}"

		FILE_NAME_LIST=("$DOWNLOAD_DIR/"*".m4b")
		FILE_NAME="$(basename "${FILE_NAME_LIST[0]}" ".m4b")"

		if [[ -z $TITLE ]]; then
			TITLE="$FILE_NAME"
		fi

		if [[ -z $AUTHOR ]]; then
			AUTHOR="$FILE_NAME"
		fi

		if [[ -n $SUBTITLE ]]; then
			TITLE="$TITLE $SUBTITLE"
		fi

		OUTPUT_FULL_DIR="$OUTPUT_DIR/$AUTHOR/$TITLE $WORK_ITEM"

		mkdir -p "$OUTPUT_FULL_DIR"

		if compgen -G "$DOWNLOAD_DIR/*.pdf" >/dev/null; then
			mv -v "$DOWNLOAD_DIR/"*".pdf" "$OUTPUT_FULL_DIR"
		fi

		mv -v "$DOWNLOAD_DIR/"*".jpg" "$OUTPUT_FULL_DIR"
		mv -v "$DOWNLOAD_DIR/"*".m4b" "$OUTPUT_FULL_DIR"
		rm -vrf "${DOWNLOAD_DIR:?}/"*
		echo "$WORK_ITEM" >>"$DATA_DIR/$PROFILE-library"
		WORKLIST_SIZE="$((WORKLIST_SIZE - 1))"

		echo "▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒"
		echo "Queue now $WORKLIST_SIZE of $PROFILE library size $LIBRARY_SIZE"
	done

	mv -v "$TEMP_DIR/library-list.new" "$DATA_DIR/$PROFILE-library"
	echo "██████████████████████████████████████████████████████████████████████████████████"
done

[mkb79]

Good work! Thank you. Do you have a ready dockerfile you can provide to me?

[PurpleBooth]

Sure

FROM ubuntu

RUN apt-get update && apt-get upgrade -y && apt-get install -y git python3 python3-pip ffmpeg
RUN git clone https://github.com/PurpleBooth/audible-cli.git /src && pip install /src
COPY entrypoint.sh /entrypoint.sh
RUN groupadd -g 568 nonroot
RUN useradd -u 568 -g 568 nonroot
USER nonroot



ENTRYPOINT ["/entrypoint.sh"]

And an example CronJob.

apiVersion: batch/v1
kind: CronJob
metadata:
  labels:
    app.kubernetes.io/instance: audible-cli
    app.kubernetes.io/name: audible-cli
  name: audible-cli-import
spec:
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      backoffLimit: 100
      template:
        spec:
          securityContext:
            runAsUser: 568
            runAsGroup: 568
            fsGroup: 568
            fsGroupChangePolicy: OnRootMismatch
          imagePullSecrets:
          - name: github-image-auth
          affinity:
            podAntiAffinity:
              preferredDuringSchedulingIgnoredDuringExecution:
              - podAffinityTerm:
                  labelSelector:
                    matchExpressions:
                    - key: app.kubernetes.io/name
                      operator: In
                      values:
                      - plex
                  topologyKey: topology.kubernetes.io/zone
                weight: 100
          containers:
          - env:
            - name: TZ
              value: Europe/Berlin
            - name: CONFIG_DIR
              value: /config
            - name: DATA_DIR
              value: /data
            - name: OUTPUT
              value: /output
            - name: TEMP_DIR
              value: /ephemeral
            image: ghcr.io/YOUR/REPO:HERE
            imagePullPolicy: Always
            name: download
            resources:
              requests:
                memory: 400Mi
                cpu: 260m
              limits:
                memory: 4Gi
            volumeMounts:
            - mountPath: /output
              name: output
            - mountPath: /config
              name: config
            - mountPath: /data
              name: data
            - mountPath: /ephemeral
              name: ephemeral
          restartPolicy: OnFailure
          volumes:
          - name: config
            secret:
              secretName: audible-cli-config
          - name: output
            persistentVolumeClaim:
              claimName: better-than-the-audible-app
          - name: data
            persistentVolumeClaim:
              claimName: audible-cli-data
          - name: ephemeral
            emptyDir: {}

  schedule: '@hourly'