add pad to chapters

Related to #4

References:
- #4

toonily-dl.go

@@ -12,10 +12,11 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 var (
-	regex  = regexp.MustCompile(`\bhttps:\/\/cdn.\b.*\.jpg`)
+	regex  = regexp.MustCompile(`\bhttps:\/\/cdn.\b.*\.(?i)(jpg|jpeg|png|webp)`)
 	client = http.Client{}
 	usage  = `
 SYNOPSIS
@@ -33,22 +34,30 @@ FLAGS
 `
 )
 
-func main() {
-	url := os.Args[len(os.Args)-1]
-	if url == "" {
-		fmt.Printf("invalid argument: URL\nexample: toonily-dl <URL>\n")
-		os.Exit(1)
-	}
+type Book struct {
+	Title       string
+	Description string
+	Cover       string
+	Chapters    []string
+}
 
+func main() {
 	flagC := flag.String("c", "", "Indicate the chapters's list to download")
 	flagH := flag.Bool("h", false, "Print help message")
+	flagD := flag.Bool("d", false, "Save description")
 	flag.Parse()
 
 	if *flagH {
 		fmt.Println(usage)
 		os.Exit(0)
 	}
 
+	url := os.Args[len(os.Args)-1]
+	if url == "" {
+		fmt.Printf("invalid argument: URL\nexample: toonily-dl <URL>\n")
+		os.Exit(1)
+	}
+
 	chapterRange := [2]int{}
 	for idx, v := range strings.Split(*flagC, ":") {
 		n, err := strconv.Atoi(v)
@@ -66,10 +75,9 @@ func main() {
 	scanner := bufio.NewScanner(bytes.NewReader(Wget(url)))
 	scanner.Split(bufio.ScanLines)
 
-	var title string
-	var chapters []string
+	book := Book{}
 	var chapterSection bool
-	var cover string
+	var summarySection bool
 
 	for scanner.Scan() {
 		line := scanner.Text()
@@ -78,58 +86,86 @@ func main() {
 			// 9 <== `<a href="`
 			// 3 <== `/">`
 			link := line[9 : len(line)-3]
-			n, _ := strconv.Atoi(link[strings.LastIndex(link, "-")+1:])
+			n, err := strconv.Atoi(link[strings.LastIndex(link, "-")+1:])
+			if err != nil {
+				fmt.Printf("Unexpected chapter %q is about to be downloaded\n", link[strings.LastIndex(link, "/")+1:])
+			}
 			between := chapterRange[0] <= n && n <= chapterRange[1]
 			exact := chapterRange[0] == n && chapterRange[1] == 0
-			if between || exact || *flagC == "" {
-				chapters = append(chapters, link)
+			if between || exact || *flagC == "" || err != nil {
+				book.Chapters = append(book.Chapters, link)
 			}
 			chapterSection = false
 			continue
 		}
 		chapterSection = strings.Contains(line, "<li class=\"wp-manga-chapter")
-		if cover == "" {
+
+		if summarySection {
+			book.Description = line[4 : len(line)-4]
+			summarySection = false
+			continue
+		}
+		if book.Description == "" {
+			summarySection = strings.Contains(line, "summary__content")
+		}
+		if book.Cover == "" {
 			if sIdx := strings.Index(line, `data-src="`); sIdx > 0 {
 				sIdx += 10 // 10 <== `data-src="`
 				if eIdx := strings.Index(line[sIdx:], `"`); eIdx > 0 {
-					cover = line[sIdx : sIdx+eIdx]
+					book.Cover = line[sIdx : sIdx+eIdx]
 				}
 			}
 		}
 
-		if title == "" {
+		if book.Title == "" {
 			if strings.HasPrefix(line, "<title>") {
 				// 12 <== `<title>Read `
 				// 24 <== ` Manga - Toonily</title>`
-				title = line[12 : len(line)-24]
+				book.Title = line[12 : len(line)-24]
+				book.Title = strings.ReplaceAll(book.Title, "&#8217;", "'")
+				book.Title = strings.ReplaceAll(book.Title, "&#8230;", "...")
 			}
 		}
 	}
 
-	MakeDir(title)
-	os.Chdir(title)
-	DownloadCover(cover)
+	MakeDir(book.Title)
+	os.Chdir(book.Title)
+	DownloadCover(book.Cover)
+
+	if *flagD {
+		SaveTextToFile(book.Description, "description.txt")
+	}
 
-	fmt.Println("Download:", title)
-	for _, url := range chapters {
+	fmt.Println("Download:", book.Title)
+	for _, url := range book.Chapters {
 		sl := strings.Split(url, "/")
-		chapter := sl[len(sl)-1]
+		slen := len(sl)
+		chapter := fmt.Sprintf("chapter-%03s", sl[len(sl)-1][8:]) // 8 <== `chapter-`
+		if !strings.HasPrefix(sl[slen-1], "chapter-") {
+			chapter = sl[slen-1] // unexpected chapter
+		}
 		MakeDir(chapter)
 		os.Chdir(chapter)
 
 		fmt.Println("Working on", chapter)
 		page := bufio.NewScanner(bytes.NewReader(Wget(url)))
 		page.Split(bufio.ScanLines)
 
+		var wg sync.WaitGroup
 		for page.Scan() {
 			img := regex.FindString(page.Text())
 			if len(img) == 0 {
 				continue
 			}
-			if err := DownloadFile(img); err != nil {
-				fmt.Println("[error]", err)
-			}
+			wg.Add(1)
+			go func(wg *sync.WaitGroup, img string) {
+				defer wg.Done()
+				if err := DownloadFile(img); err != nil {
+					fmt.Println("[error]", err)
+				}
+			}(&wg, img)
 		}
+		wg.Wait()
 		os.Chdir("../")
 
 		if cnt, err := os.ReadDir(chapter); err == nil {
@@ -215,3 +251,22 @@ func DownloadCover(url string) {
 		}
 	}
 }
+
+func SaveTextToFile(txt, dst string) {
+	if _, err := os.Stat(dst); err == nil {
+		return
+	}
+
+	file, err := os.Create(dst)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer file.Close()
+
+	txt = strings.ReplaceAll(txt, "s/&#8217;", "'")
+	txt = strings.ReplaceAll(txt, "&#8230;", "...")
+	if _, err = file.WriteString(txt); err != nil {
+		fmt.Println(err)
+	}
+}

toonily-dl.sh

@@ -1,16 +1,21 @@
- #!/bin/bash
+#!/bin/bash
+set -e
 
-if [[ $# -eq 0 || -z "$1" ]]; then
+total_args=$#
+last_arg=${!total_args}
+
+if [[ $total_args -eq 0 || -z "$last_arg" ]]; then
 	echo "invalid argument: URL"
 	echo "example: toonily-dl.sh https://toonily.com/webtoon/amazing-manga/" 
 	exit 1
 fi
-toon=$(curl -s "$1")
+toon=$(curl -s "$last_arg")
 
 Title_Line=$(echo "$toon" | grep -E "^<title>")
-start_idx=$(expr length "<title>Read ")
-end_idx=$(expr length " Manga - Toonily</title>")
+start_idx=12 # 12 <== `<title>Read `
+end_idx=24   # 24 <== ` Manga - Toonily</title>`
 title=${Title_Line:$start_idx:-$end_idx}
+title=$(echo "$title"|sed 's/&#8217;/’/g'|sed 's/&#8230;/.../g')
 
 if [ ! -d "$title" ]; then
 	mkdir "$title"
@@ -19,55 +24,42 @@ cd "$title" || exit
 echo "Download: $title"
 
 if [ ! -e "cover.jpg" ]; then
-	wget -qc $(echo "$toon" | grep -oP 'data-src="\K[^"]+') -o "cover.jpg"
+	wget -qc "$(echo "$toon" | grep -oP 'data-src="\K[^"]+')" -O "cover.jpg"
 fi
 
-while getopts "c:" flag; do
+while getopts ":dc:" flag; do
 	case ${flag} in
-		-c)  flag_chapter=$OPTARG	;;
+		c) flag_chapter=$OPTARG	;;
+		d) description=$(echo "$toon"|grep -A1 -w "summary__content"|grep "<p>")
+		   description=$(echo "$description"|sed -n 's|<p>\(.*\)</p>|\1|p')
+		   description=$(echo "$description"|sed 's/&#8217;/’/g'|sed 's/&#8230;/.../g')
+		   echo "$description" > "description.txt"
+		   ;; 
 		*)  echo "Usage: toonily-dl.sh [options] <URL>"; exit 1	;;
 	esac
 done
 
-# while [[ "$#" -gt 0 ]]; do
-#     case $1 in
-#         -c)
-#             if [[ $2 == *":"* ]]; then
-#                 start=$(echo $2 | cut -d':' -f1)
-#                 end=$(echo $2 | cut -d':' -f2)
-#                 echo "Range detected: $start to $end"
-#             elif [[ $2 == *":" ]]; then
-#                 start=""
-#                 end=$(echo $2 | cut -d':' -f1)
-#                 echo "End range detected: up to $end"
-#             else
-#                 echo "Single value detected: $2"
-#             fi
-#             shift
-#             ;;
-#         *)
-#             echo "Unknown parameter passed: $1"
-#             exit 1
-#             ;;
-#     esac
-#     shift
-# done
-
 IFS=':' read -r -a chapterRange <<< "$flag_chapter"
-if [ -z ${chapterRange[0]} ]; then chapterRange[0]="0"; fi
-if [ -z ${chapterRange[1]} ]; then chapterRange[1]="1000000"; fi
+if [ -z "${chapterRange[0]}" ]; then chapterRange[0]="0"; fi
+if [ -z "${chapterRange[1]}" ]; then chapterRange[1]="1000000"; fi
 
 chapters=$(echo "$toon" | grep -A1 -w "class=\"wp-manga-chapter" | grep href | cut -d "\"" -f2)
 for chapter in $chapters;
 do
 	chapter_dir=$(echo "$chapter" | cut -d "/" -f6)
 	idx=$(echo "$chapter_dir"|cut -d "-" -f2)
-	if [ ! $idx -ge ${chapterRange[0]} && ! $idx -le ${chapterRange[1]} ]; then
+
+	if [ ! "${idx//[0-9]}" = "" ]; then
+		echo "Unexpected chapter '$idx' is about to be downloaded"
+	elif [ ! "$idx" -ge "${chapterRange[0]}" ] && [ ! "$idx" -le "${chapterRange[1]}" ]; then
 		continue
-	elif [ ! $idx = ${chapterRange[0]} && ! ${chapterRange[1]} = 0 ]; then
+	elif [ ! "$idx" = "${chapterRange[0]}" ] && [ ! "${chapterRange[1]}" = 0 ]; then
 		continue
 	fi
 
+	if [ "${idx//[0-9]}" = "" ]; then
+		chapter_dir=$(printf "chapter-%03d" "$idx")
+	fi
 	if [ ! -d "$chapter_dir" ]; then
 		mkdir "$chapter_dir"
 	fi
@@ -79,7 +71,7 @@ do
 	do
 		wget --quiet --header 'authority: cdn.toonily.com' --header 'referer: https://toonily.com/' --continue "$img"
 	done
-	echo "Downloaded $(ls -1|wc -l) file(s)"
+	echo "Downloaded $(find . -type f|wc -l) file(s)"
 
 	# Exit from chapter directory
 	cd ../