Skip to content

Commit

Permalink
Preview caching (#36)
Browse files Browse the repository at this point in the history 
* Minor changes

* Added proxy type option

* Fix option that enables DNS resolving over proxy

* Caches image previews

* Must cache properly now

* Changed permissions

* Fix generating thumbs if keep_thumbs is disabled

Co-authored-by: luna <luna@Moon>
  • Loading branch information
@ada-af
ada-af and luna committed Sep 2, 2020
1 parent 4844c93 commit 8cfb826
Show file tree
Hide file tree
Showing 11 changed files with 131 additions and 91 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,5 @@ __pycache__/settings_file.cpython-36.pyc
sqlite.db
.idea/**
.vscode/**
thumbs/**
dermod/sitesupport/__pycache__/**
20 changes: 16 additions & 4 deletions DBooru_web.py
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,19 @@
from dermod import input_parser as ip
from dermod import mime_types as mimes
from dermod import predict
from dermod.helpers import Option, Module_Options
from dermod.helpers import Option, Module_Options, ThumbFile

try:
import PIL.Image as Image
except ImportError:
pass

if settings_file.keep_thumbs:
try:
os.makedirs(settings_file.thumbs_path)
except FileExistsError:
pass

try:
os.remove("update.lck")
except Exception:
Expand Down Expand Up @@ -148,10 +154,16 @@ def encode_FFMPEG(fname, tf):

@DBooru.route("/thumbnail/<string:fname>")
def thumbnail(fname):
tf = tempfile.NamedTemporaryFile(mode="wb+", delete=False)
tf.close()
if settings_file.thumbnailer == "ffmpeg" and settings_file.keep_thumbs:
tf = ThumbFile(fname)
else:
tf = tempfile.NamedTemporaryFile(mode="wb+", delete=False)
tf.close()
if settings_file.thumbnailer.lower() == 'ffmpeg':
encode_FFMPEG(fname, tf)
if os.path.isfile(tf.name) and settings_file.keep_thumbs:
return send_file(tf.name)
else:
encode_FFMPEG(fname, tf)
elif settings_file.thumbnailer.lower() == 'pil':
encode_PIL(fname, tf)
else:
Expand Down
71 changes: 36 additions & 35 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,35 +337,36 @@ Enter this commands if prompt starts with `Search@DB>`
## Settings_file.py

| Option | Format | Description |
| ----------------- | ------------------------------------------- | --------------------------------------------------------------------------- |
| modules | List (['String', 'String']) | Enables modules |
| suppress_errors | Bool (True/False) | Prints errors and stacktrace in case of happening |
| ssl_verify | Bool (True/False) or String ("Path") | Enable/Disable ssl verification or set custom CA Cert |
| enable_proxy | Bool (True/False) | Enables/Disables proxy for requests |
| socks5_proxy_ip | String (IP) | Sets proxy IP |
| socks5_proxy_port | String (Port) | Sets proxy port |
| BASE_DIR | Function | Magic for flask to work |
| web_ip | String (IP) | Set IP to bind Web interface |
| web_port | Integer (port) | Sets port to bind Web interface |
| thumbnailer | String (One of "ffmpeg", "PIL", "disabled") | Defines tool to make thumbnails or not to make them at all |
| conv_format | String (ffmpeg output format) | Format to use when making thumbnails |
| ffmpeg_args | String (ffmpeg parameters) | For situations when you think that default settings suck |
| gif_to_webp | Bool (True/False) | Creates webp thumbnails for gifs |
| disable_mobile | Bool (True/False) | Should tag prediction be disabled on mobile |
| predict_tags | Integer (number) | How many tags to show when predicting input |
| showing_imgs | Integer (number) | How many images to show per page |
| showing_tags | Integer (number) | How many tags to **show** per image (CLI-only) |
| images_path | String (Path) | Where to store loaded images |
| export_path | String (Path) | Where to store exported images |
| time_wait | Integer (seconds) | How long thread can stay alive |
| ids_file | String (Path/Filename) | Name for tempfile (No need to change) |
| db_name | String (Path/Filename) | Where to store DB file |
| thread_cap | Integer (number) | Defines maximum running threads before blocking creating new threads |
| sleep_time | Integer (seconds) | Defines time to wait before creating new thread after thread cap is reached |
| enable_polling | Bool | Setting for enabling/disabling polling for changes in settings_file.py |
| polling_time | Integer (seconds) | Defines time (in seconds) between checks for changes in settings_file.py |
| first_run | Bool | Defines if settings page should be opened on first run |
| Option | Format | Description |
| --------------- | ------------------------------------------- | --------------------------------------------------------------------------- |
| modules | List (['String', 'String']) | Enables modules |
| suppress_errors | Bool (True/False) | Prints errors and stacktrace in case of happening |
| ssl_verify | Bool (True/False) or String ("Path") | Enable/Disable ssl verification or set custom CA Cert |
| enable_proxy | Bool (True/False) | Enables/Disables proxy for requests |
| proxy_type | String | Sets proxy type |
| proxy_ip | String (IP) | Sets proxy IP |
| proxy_port | String (Port) | Sets proxy port |
| BASE_DIR | Function | Magic for flask to work |
| web_ip | String (IP) | Set IP to bind Web interface |
| web_port | Integer (port) | Sets port to bind Web interface |
| thumbnailer | String (One of "ffmpeg", "PIL", "disabled") | Defines tool to make thumbnails or not to make them at all |
| conv_format | String (ffmpeg output format) | Format to use when making thumbnails |
| ffmpeg_args | String (ffmpeg parameters) | For situations when you think that default settings suck |
| gif_to_webp | Bool (True/False) | Creates webp thumbnails for gifs |
| disable_mobile | Bool (True/False) | Should tag prediction be disabled on mobile |
| predict_tags | Integer (number) | How many tags to show when predicting input |
| showing_imgs | Integer (number) | How many images to show per page |
| showing_tags | Integer (number) | How many tags to **show** per image (CLI-only) |
| images_path | String (Path) | Where to store loaded images |
| export_path | String (Path) | Where to store exported images |
| time_wait | Integer (seconds) | How long thread can stay alive |
| ids_file | String (Path/Filename) | Name for tempfile (No need to change) |
| db_name | String (Path/Filename) | Where to store DB file |
| thread_cap | Integer (number) | Defines maximum running threads before blocking creating new threads |
| sleep_time | Integer (seconds) | Defines time to wait before creating new thread after thread cap is reached |
| enable_polling | Bool | Setting for enabling/disabling polling for changes in settings_file.py |
| polling_time | Integer (seconds) | Defines time (in seconds) between checks for changes in settings_file.py |
| first_run | Bool | Defines if settings page should be opened on first run |

### /settings autogeneration

Expand All @@ -383,12 +384,12 @@ By default examples are NOT shown on page

Keywords must be followed by `:` , except for `# Require example`

| Keyword | Description | Notes |
|-|-| - |
| Example | Defines example | Can be used more than once |
| Format | Defines variable type | Should be one of `string`, `bool`, `int`. Defaults to `string`.
| Options | Defines available options for variable | Must be formatted as Python list
| Require example | Requires examples to be shown on /settings page | May or may not end with `:` |
| Keyword | Description | Notes |
| --------------- | ----------------------------------------------- | --------------------------------------------------------------- |
| Example | Defines example | Can be used more than once |
| Format | Defines variable type | Should be one of `string`, `bool`, `int`. Defaults to `string`. |
| Options | Defines available options for variable | Must be formatted as Python list |
| Require example | Requires examples to be shown on /settings page | May or may not end with `:` |

#### Examples

Expand Down
6 changes: 6 additions & 0 deletions dermod/helpers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ def _disassemble(self, data):
if i:
self.options[i.split('=')[0].strip()] = i.split('=')[1].split('#')[0].strip()


class ThumbFile:
def __init__(self, fname):
self.name = settings_file.thumbs_path+f"/thumb_{fname}"


class Option:

def __init__(self, data):
Expand Down
12 changes: 6 additions & 6 deletions dermod/imgloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def get_raw_image(self):
else:
self.tmp = s.get(
"{}".format(self.url),
proxies=dict(https='socks5://{}:{}'.format(self.ip, self.port),
http='socks5://{}:{}'.format(self.ip, self.port)), verify=settings_file.ssl_verify)
proxies=dict(https='{}://{}:{}'.format(settings_file.proxy_type, self.ip, self.port),
http='{}://{}:{}'.format(settings_file.proxy_type, self.ip, self.port)), verify=settings_file.ssl_verify)
if self.tmp.status_code >= 400:
global is_error_code
is_error_code = True
Expand Down Expand Up @@ -101,8 +101,8 @@ def run(module, file, check_files=True, check_local=True, endwith="\r"):
str(parsed[i][7] + parsed[i][0]),
parsed[i][1],
settings_file.enable_proxy,
settings_file.socks5_proxy_ip,
settings_file.socks5_proxy_port)
settings_file.proxy_ip,
settings_file.proxy_port)
t.start()
tc.threads.append(t)
time.sleep(slp)
Expand All @@ -122,8 +122,8 @@ def run(module, file, check_files=True, check_local=True, endwith="\r"):
str(parsed[i][7] + parsed[i][0]),
parsed[i][1],
settings_file.enable_proxy,
settings_file.socks5_proxy_ip,
settings_file.socks5_proxy_port)
settings_file.proxy_ip,
settings_file.proxy_port)
t.start()
tc.threads.append(t)
time.sleep(module.slp)
Expand Down
12 changes: 6 additions & 6 deletions dermod/listloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ def get_data(self):
params=self.module.params,
paginator=self.module.paginator.format(self.page)),
proxies=dict(
https='socks5://{}:{}'.format(
settings_file.socks5_proxy_ip, settings_file.socks5_proxy_port),
http='socks5://{}:{}'.format(settings_file.socks5_proxy_ip, settings_file.socks5_proxy_port)),
https='{}://{}:{}'.format(
settings_file.proxy_type, settings_file.proxy_ip, settings_file.proxy_port),
http='{}://{}:{}'.format(settings_file.proxy_type, settings_file.proxy_ip, settings_file.proxy_port)),
verify=settings_file.ssl_verify, timeout=settings_file.time_wait)
if self.raw_data.status_code >= 400:
is_error_code = True
Expand Down Expand Up @@ -140,9 +140,9 @@ def run(module, pages_num=0, file=settings_file.ids_file, endwith="\r"):
params=module.params,
paginator=module.paginator.format(pages_num)),
proxies=dict(
https='socks5://{}:{}'.format(
settings_file.socks5_proxy_ip, settings_file.socks5_proxy_port),
http='socks5://{}:{}'.format(settings_file.socks5_proxy_ip, settings_file.socks5_proxy_port)),
https='{}://{}:{}'.format(
settings_file.proxy_type, settings_file.proxy_ip, settings_file.proxy_port),
http='{}://{}:{}'.format(settings_file.proxy_type, settings_file.proxy_ip, settings_file.proxy_port)),
verify=settings_file.ssl_verify, timeout=settings_file.time_wait)
else:
dat = s.get(
Expand Down
2 changes: 1 addition & 1 deletion dermod/sitesupport/derpibooru.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,5 @@ def parse(self, string, pg_num):
self.height.append(height)
self.form.append(form)
self.links.append(url)
self.tags.append(tags)
self.tags.append(tags.replace('[', '').replace(']', ''))
j += 1
2 changes: 1 addition & 1 deletion dermod/sitesupport/e621.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def parse(self, string):
tags += j

k = r + ",," + ",,".join(sorted(tags))
self.tags.append(k)
self.tags.append(k.replace('[', '').replace(']', ''))

k = i.split('"file":')[1]

Expand Down
8 changes: 4 additions & 4 deletions dermod/sitesupport/gelbooru.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ def _get_data(self):
raw_data = s.get(
"https://gelbooru.com/index.php?page=dapi&s=post&q=index&json=1&id={id}".format(id=self.id),
proxies=dict(
https='socks5://{}:{}'.format(
self.settings_file.socks5_proxy_ip, self.settings_file.socks5_proxy_port),
http='socks5://{}:{}'.format(self.settings_file.socks5_proxy_ip, self.settings_file.socks5_proxy_port)),
https='{}://{}:{}'.format(
self.settings_file.proxy_type, self.settings_file.proxy_ip, self.settings_file.proxy_port),
http='{}://{}:{}'.format(self.settings_file.proxy_type, self.settings_file.proxy_ip, self.settings_file.proxy_port)),
verify=self.settings_file.ssl_verify, timeout=self.settings_file.time_wait)
json = self.json.loads(raw_data.text)
self.links = json[0]['file_url']
Expand All @@ -87,6 +87,6 @@ def _get_data(self):
elif r == "s":
r = "safe"
self.width = str(json[0]['width'])
self.tags = ",,".join(x.replace("_", " ") for x in str(json[0]['tags']).split(" "))
self.tags = ",,".join(x.replace("_", " ").replace('[', '').replace(']', '') for x in str(json[0]['tags']).split(" "))
self.tags = self.tags + ',,' + r
self.ready = True
Loading

0 comments on commit 8cfb826

Please sign in to comment.