From 50dc79458b510798af2c8fca090e78de7032836b Mon Sep 17 00:00:00 2001 From: "Alexander G. Morano" Date: Sun, 21 Jul 2024 17:58:36 -0700 Subject: [PATCH] api routes for glsl programs tracker for glsl programs as nodes reorg of utils.py nicer globals for JS side --- __init__.py | 59 +++++++++++- core/create.py | 29 +++--- res/glsl/basic.glsl | 9 ++ res/glsl/basic.vert | 6 ++ sup/image.py | 180 +++++++++++++++++++++++++++++++---- sup/lexicon.py | 5 +- sup/shader.py | 85 +++++++++++------ sup/util.py | 182 ++++++++++++++++++------------------ web/core/core_cozy_menu.js | 32 ++++--- web/nodes/delay.js | 12 ++- web/nodes/glsl.js | 17 ++-- web/nodes/queue.js | 16 +++- web/nodes/tick.js | 7 +- web/util/util_api.js | 14 +++ web/widget/widget_vector.js | 2 +- 15 files changed, 475 insertions(+), 180 deletions(-) create mode 100644 res/glsl/basic.glsl create mode 100644 res/glsl/basic.vert diff --git a/__init__.py b/__init__.py index d16c5a0..00e5130 100644 --- a/__init__.py +++ b/__init__.py @@ -69,10 +69,26 @@ JOV_WEB = ROOT / 'web' JOV_DEFAULT = JOV_WEB / 'default.json' JOV_CONFIG_FILE = JOV_WEB / 'config.json' +# +JOV_ROOT_GLSL = ROOT / 'res' / 'glsl' +GLSL_PROGRAMS = { + "vertex": { "NONE": None }, + "fragment": { "NONE": None } +} + +GLSL_PROGRAMS["vertex"].update({str(f.name): str(f) for f in Path(JOV_ROOT_GLSL).glob('*.vert')}) +if (USER_GLSL := os.getenv("JOV_GLSL", None)) is not None: + GLSL_PROGRAMS["vertex"].update({str(f.name): str(f) for f in Path(USER_GLSL).glob('*.vert')}) + +GLSL_PROGRAMS["fragment"].update({str(f.name): str(f) for f in Path(JOV_ROOT_GLSL).glob('*.glsl')}) +if USER_GLSL is not None: + GLSL_PROGRAMS["fragment"].update({str(f.name): str(f) for f in Path(USER_GLSL).glob('*.glsl')}) + +logger.info(f"found {len(GLSL_PROGRAMS['vertex'])} vertex and {len(GLSL_PROGRAMS['fragment'])} fragment programs") +print(GLSL_PROGRAMS) # nodes to skip on import; for online systems; skip Export, Streamreader, etc... JOV_IGNORE_NODE = ROOT / 'ignore.txt' -JOV_GLSL = ROOT / 'res' / 'glsl' JOV_SIDECAR = os.getenv("JOV_SIDECAR", str(ROOT / "_md")) JOV_LOG_LEVEL = os.getenv("JOV_LOG_LEVEL", "WARNING") @@ -142,6 +158,17 @@ def __ne__(self, __value: object) -> bool: # == API RESPONSE # ============================================================================= +def load_file(fname: str) -> Any: + try: + with open(fname, 'r') as f: + return f.read() + except Exception as e: + logger.error(e) + +# ============================================================================= +# == API RESPONSE +# ============================================================================= + class TimedOutException(Exception): pass class ComfyAPIMessage: @@ -236,6 +263,36 @@ async def jovimetrix_doc(request) -> Any: f.write(data[k]['.md']) return web.json_response(data) + @PromptServer.instance.routes.get("/jovimetrix/glsl") + async def jovimetrix_glsl_list(request) -> Any: + ret = {k:[kk for kk, vv in v.items() \ + if kk not in ['NONE'] and vv not in [None] and Path(vv).exists()] + for k, v in GLSL_PROGRAMS.items()} + return web.json_response(ret) + + @PromptServer.instance.routes.get("/jovimetrix/glsl/{shader}") + async def jovimetrix_glsl_raw(request, shader:str) -> Any: + if (program := GLSL_PROGRAMS.get(shader, None)) is None: + return web.json_response(f"no program {shader}") + response = load_file(program) + return web.json_response(response) + + @PromptServer.instance.routes.post("/jovimetrix/glsl") + async def jovimetrix_glsl(request) -> Any: + json_data = await request.json() + response = {k:None for k in json_data.keys()} + for who in response.keys(): + if (programs := GLSL_PROGRAMS.get(who, None)) is None: + logger.warning(f"no program type {who}") + continue + fname = json_data[who] + if (data := programs.get(fname, None)) is not None: + response[who] = load_file(data) + else: + logger.warning(f"no glsl shader entry {fname}") + + return web.json_response(response) + except Exception as e: logger.error(e) diff --git a/core/create.py b/core/create.py index b091919..d7b6662 100644 --- a/core/create.py +++ b/core/create.py @@ -13,15 +13,16 @@ from comfy.utils import ProgressBar -from Jovimetrix import comfy_message, parse_reset, JOVBaseNode, ROOT, JOV_TYPE_IMAGE +from Jovimetrix import comfy_message, parse_reset, JOVBaseNode, \ + JOV_TYPE_IMAGE, GLSL_PROGRAMS from Jovimetrix.sup.lexicon import JOVImageNode, Lexicon from Jovimetrix.sup.util import parse_param, zip_longest_fill, EnumConvertType -from Jovimetrix.sup.image import channel_solid, cv2tensor, cv2tensor_full, image_convert, \ - image_grayscale, image_invert, image_mask_add, pil2cv, tensor2pil, \ - image_rotate, image_scalefit, image_stereogram, image_transform, image_translate, \ - pixel_eval, tensor2cv, shape_ellipse, shape_polygon, shape_quad, \ +from Jovimetrix.sup.image import channel_solid, cv2tensor, cv2tensor_full, \ + image_grayscale, image_invert, image_mask_add, pil2cv, image_convert, \ + image_rotate, image_scalefit, image_stereogram, image_transform, \ + tensor2cv, shape_ellipse, shape_polygon, shape_quad, image_translate, \ EnumScaleMode, EnumInterpolation, EnumEdge, EnumImageType, MIN_IMAGE_SIZE from Jovimetrix.sup.text import font_names, text_autosize, text_draw, \ @@ -33,7 +34,6 @@ # ============================================================================= JOV_CATEGORY = "CREATE" -JOV_CONFIG_GLSL = ROOT / 'glsl' # ============================================================================= @@ -94,6 +94,7 @@ class GLSLNode(JOVImageNode): DESCRIPTION = """ Execute custom GLSL (OpenGL Shading Language) fragment shaders to generate images or apply effects. GLSL is a high-level shading language used for graphics programming, particularly in the context of rendering images or animations. This node allows for real-time rendering of shader effects, providing flexibility and creative control over image processing pipelines. It takes advantage of GPU acceleration for efficient computation, enabling the rapid generation of complex visual effects. """ + INSTANCE = 0 @classmethod def INPUT_TYPES(cls) -> dict: @@ -101,14 +102,15 @@ def INPUT_TYPES(cls) -> dict: d.update({ "optional": { Lexicon.TIME: ("FLOAT", {"default": 0, "step": 0.001, "min": 0, "precision": 4}), - Lexicon.BATCH: ("INT", {"default": 1, "step": 1, "min": 0, "max": 262144}), + Lexicon.BATCH: ("INT", {"default": 0, "step": 1, "min": 0, "max": 1048576}), Lexicon.FPS: ("INT", {"default": 24, "step": 1, "min": 1, "max": 120}), Lexicon.WH: ("VEC2", {"default": (512, 512), "min": MIN_IMAGE_SIZE, "step": 1,}), Lexicon.MATTE: ("VEC4", {"default": (0, 0, 0, 255), "step": 1, "label": [Lexicon.R, Lexicon.G, Lexicon.B, Lexicon.A], "rgb": True}), Lexicon.WAIT: ("BOOLEAN", {"default": False}), Lexicon.RESET: ("BOOLEAN", {"default": False}), - Lexicon.FRAGMENT: ("STRING", {"default": GLSLShader.PROG_FRAGMENT, "multiline": True, "dynamicPrompts": False}), + Lexicon.PROG_VERT: ("STRING", {"default": GLSLShader.PROG_VERTEX, "multiline": True, "dynamicPrompts": False}), + Lexicon.PROG_FRAG: ("STRING", {"default": GLSLShader.PROG_FRAGMENT, "multiline": True, "dynamicPrompts": False}), } }) return Lexicon._parse(d, cls) @@ -130,17 +132,18 @@ def run(self, ident, **kw) -> tuple[torch.Tensor]: reset = parse_param(kw, Lexicon.RESET, EnumConvertType.BOOLEAN, False)[0] wihi = parse_param(kw, Lexicon.WH, EnumConvertType.VEC2INT, [(512, 512)], MIN_IMAGE_SIZE)[0] matte = parse_param(kw, Lexicon.MATTE, EnumConvertType.VEC4INT, [(0, 0, 0, 255)], 0, 255)[0] - fragment = parse_param(kw, Lexicon.FRAGMENT, EnumConvertType.STRING, GLSLShader.PROG_FRAGMENT)[0] + vertex_src = parse_param(kw, Lexicon.PROG_VERT, EnumConvertType.STRING, "")[0] + fragment_src = parse_param(kw, Lexicon.PROG_FRAG, EnumConvertType.STRING, "")[0] variables = kw.copy() - for p in [Lexicon.TIME, Lexicon.BATCH, Lexicon.FPS, Lexicon.WH, Lexicon.FRAGMENT, Lexicon.WAIT, Lexicon.RESET]: + for p in [Lexicon.TIME, Lexicon.BATCH, Lexicon.FPS, Lexicon.WAIT, Lexicon.RESET, Lexicon.WH, Lexicon.MATTE, Lexicon.PROG_VERT, Lexicon.PROG_FRAG]: variables.pop(p, None) self.__glsl.bgcolor = matte self.__glsl.size = wihi self.__glsl.fps = fps try: - self.__glsl.fragment = fragment + self.__glsl.program(vertex_src, fragment_src) except CompileException as e: comfy_message(ident, "jovi-glsl-error", {"id": ident, "e": str(e)}) logger.error(e) @@ -168,8 +171,8 @@ def run(self, ident, **kw) -> tuple[torch.Tensor]: images.append(cv2tensor_full(image)) if not wait: self.__delta += step - if batch == 0: - comfy_message(ident, "jovi-glsl-time", {"id": ident, "t": self.__delta}) + # if batch == 0: + comfy_message(ident, "jovi-glsl-time", {"id": ident, "t": self.__delta}) pbar.update_absolute(idx) return [torch.cat(i, dim=0) for i in zip(*images)] diff --git a/res/glsl/basic.glsl b/res/glsl/basic.glsl new file mode 100644 index 0000000..e022fe4 --- /dev/null +++ b/res/glsl/basic.glsl @@ -0,0 +1,9 @@ +uniform sampler2D imageA; +uniform sampler2D imageB; + +void mainImage( out vec4 fragColor, vec2 fragCoord ) { + vec2 uv = fragCoord.xy / iResolution.xy; + vec3 col = texture2D(imageA, uv).rgb; + vec3 col2 = texture2D(imageB, uv).rgb; + fragColor = vec4(mix(col, col2, 0.5), 1.0); +} \ No newline at end of file diff --git a/res/glsl/basic.vert b/res/glsl/basic.vert new file mode 100644 index 0000000..deab4c0 --- /dev/null +++ b/res/glsl/basic.vert @@ -0,0 +1,6 @@ +#version 330 core +void main() +{ + vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3)); + gl_Position = vec4(verts[gl_VertexID], 0, 1); +} \ No newline at end of file diff --git a/sup/image.py b/sup/image.py index a8c686b..5386169 100644 --- a/sup/image.py +++ b/sup/image.py @@ -1143,14 +1143,14 @@ def image_gradient_map(image:TYPE_IMAGE, gradient_map:TYPE_IMAGE, reverse:bool=F def image_grayscale(image:TYPE_IMAGE) -> TYPE_IMAGE: if image.dtype in [np.float16, np.float32, np.float64]: - image = np.clip(image * 255, 0, 255).astype(np.uint8) - cc = image.shape[2] if image.ndim == 3 else 1 - if cc == 1: - return image - if cc == 4: - image = image[:,:,:3] - image = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)[:,:,2] - # WxH to WxHx1 + # normalize + image = (image - image.min()) / (image.max() - image.min()) + image = (image * 255).astype(np.uint8) + if image.ndim == 3: + if image.shape[2] == 3: + image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + else: + image = cv2.cvtColor(image, cv2.COLOR_BGRA2GRAY) return np.expand_dims(image, -1) def image_grid(data: List[TYPE_IMAGE], width: int, height: int) -> TYPE_IMAGE: @@ -1217,29 +1217,28 @@ def image_invert(image: TYPE_IMAGE, value: float) -> TYPE_IMAGE: image = cv2.addWeighted(image, 1 - value, 255 - image, value, 0) return bgr2image(image, alpha, cc == 1) -def image_lerp(imageA:TYPE_IMAGE, - imageB:TYPE_IMAGE, - mask:TYPE_IMAGE=None, - alpha:float=1.) -> TYPE_IMAGE: +def image_lerp(imageA:TYPE_IMAGE, imageB:TYPE_IMAGE, mask:TYPE_IMAGE=None, + alpha:float=1.) -> TYPE_IMAGE: imageA = imageA.astype(np.float32) imageB = imageB.astype(np.float32) - # normalize alpha and establish mask + # establish mask alpha = np.clip(alpha, 0, 1) if mask is None: height, width = imageA.shape[:2] - mask = cv2.empty((height, width, 1), dtype=cv2.uint8) + mask = np.ones((height, width, 1), dtype=np.float32) else: # normalize the mask - info = np.iinfo(mask.dtype) - mask = mask.astype(np.float32) / info.max * alpha + mask = mask.astype(np.float32) + mask = (mask - mask.min()) / (mask.max() - mask.min()) * alpha # LERP imageA = cv2.multiply(1. - mask, imageA) imageB = cv2.multiply(mask, imageB) - imageA = cv2.add(imageA, imageB) - return imageA.astype(np.uint8) + imageA = (cv2.add(imageA, imageB) / 255. - 0.5) * 2.0 + imageA = (imageA * 255).astype(np.uint8) + return np.clip(imageA, 0, 255) def image_levels(image:torch.Tensor, black_point:int=0, white_point=255, mid_point=128, gamma=1.0) -> TYPE_IMAGE: @@ -1323,6 +1322,15 @@ def image_load_from_url(url:str) -> TYPE_IMAGE: except Exception as e: logger.error(str(e)) +def image_normalize(image: TYPE_IMAGE) -> TYPE_IMAGE: + image = image.astype(np.float32) + img_min = np.min(image) + img_max = np.max(image) + if img_min == img_max: + return np.zeros_like(image, dtype=np.float32) + image = (image - img_min) / (img_max - img_min) + return (image * 255).astype(np.uint8) + def image_mask(image:TYPE_IMAGE, color:TYPE_PIXEL=255) -> TYPE_IMAGE: """Returns a mask from an image or a default mask with the color.""" cc = image.shape[2] if image.ndim == 3 else 1 @@ -1438,6 +1446,19 @@ def mirror(img:TYPE_IMAGE, axis:int, reverse:bool=False) -> TYPE_IMAGE: return image +def image_mirror_mandela(imageA: np.ndarray, imageB: np.ndarray) -> Tuple[np.ndarray, ...]: + """Merge 4 flipped copies of input images to make them wrap. + Output is twice bigger in both dimensions.""" + + top = np.hstack([imageA, -np.flip(imageA, axis=1)]) + bottom = np.hstack([np.flip(imageA, axis=0), -np.flip(imageA)]) + imageA = np.vstack([top, bottom]) + + top = np.hstack([imageB, np.flip(imageB, axis=1)]) + bottom = np.hstack([-np.flip(imageB, axis=0), -np.flip(imageB)]) + imageB = np.vstack([top, bottom]) + return imageA, imageB + def image_pixelate(image: TYPE_IMAGE, amount:float=1.)-> TYPE_IMAGE: h, w = image.shape[:2] @@ -2060,3 +2081,126 @@ def remap_sphere(image: TYPE_IMAGE, radius: float) -> TYPE_IMAGE: height, width = image.shape[:2] map_x, map_y = coord_sphere(width, height, radius) return cv2.remap(image, map_x, map_y, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) + +def depth_from_gradient(grad_x, grad_y): + """Optimized Frankot-Chellappa depth-from-gradient algorithm.""" + rows, cols = grad_x.shape + rows_scale = np.fft.fftfreq(rows) + cols_scale = np.fft.fftfreq(cols) + u_grid, v_grid = np.meshgrid(cols_scale, rows_scale) + grad_x_F = np.fft.fft2(grad_x) + grad_y_F = np.fft.fft2(grad_y) + denominator = u_grid**2 + v_grid**2 + denominator[0, 0] = 1.0 + Z_F = (-1j * u_grid * grad_x_F - 1j * v_grid * grad_y_F) / denominator + Z_F[0, 0] = 0.0 + Z = np.fft.ifft2(Z_F).real + Z -= np.min(Z) + Z /= np.max(Z) + return Z + +def height_from_normal(image: TYPE_IMAGE, tile:bool=True) -> TYPE_IMAGE: + """Computes a height map from the given normal map.""" + image = np.transpose(image, (2, 0, 1)) + flip_img = np.flip(image, axis=1) + grad_x, grad_y = (flip_img[0] - 0.5) * 2, (flip_img[1] - 0.5) * 2 + grad_x = np.flip(grad_x, axis=0) + grad_y = np.flip(grad_y, axis=0) + + if not tile: + grad_x, grad_y = image_mirror_mandela(grad_x, grad_y) + pred_img = depth_from_gradient(-grad_x, grad_y) + + # re-crop + if not tile: + height, width = image.shape[1], image.shape[2] + pred_img = pred_img[:height, :width] + + image = np.stack([pred_img, pred_img, pred_img]) + image = np.transpose(image, (1, 2, 0)) + return image + +def curvature_from_normal(image: TYPE_IMAGE, blur_radius:int=2)-> TYPE_IMAGE: + """Computes a curvature map from the given normal map.""" + image = np.transpose(image, (2, 0, 1)) + blur_factor = 1 / 2 ** min(8, max(2, blur_radius)) + diff_kernel = np.array([-1, 0, 1]) + + def conv_1d(array, kernel) -> np.ndarray[Any, np.dtype[Any]]: + """Performs row-wise 1D convolutions with repeat padding.""" + k_l = len(kernel) + extended = np.pad(array, k_l // 2, mode="wrap") + return np.array([np.convolve(row, kernel, mode="valid") for row in extended[k_l//2:-k_l//2+1]]) + + h_conv = conv_1d(image[0], diff_kernel) + v_conv = conv_1d(-image[1].T, diff_kernel).T + edges_conv = h_conv + v_conv + + # Calculate blur radius in pixels + blur_radius_px = int(np.mean(image.shape[1:3]) * blur_factor) + if blur_radius_px < 2: + # If blur radius is too small, just normalize the edge convolution + image = (edges_conv - np.min(edges_conv)) / (np.ptp(edges_conv) + 1e-10) + else: + blur_radius_px += blur_radius_px % 2 == 0 + + # Compute Gaussian kernel + sigma = max(1, blur_radius_px // 8) + x = np.linspace(-(blur_radius_px - 1) / 2, (blur_radius_px - 1) / 2, blur_radius_px) + g_kernel = np.exp(-0.5 * np.square(x) / np.square(sigma)) + g_kernel /= np.sum(g_kernel) + + # Apply Gaussian blur + h_blur = conv_1d(edges_conv, g_kernel) + v_blur = conv_1d(h_blur.T, g_kernel).T + image = (v_blur - np.min(v_blur)) / (np.ptp(v_blur) + 1e-10) + + image = (image - image.min()) / (image.max() - image.min()) * 255 + return image.astype(np.uint8) + +def roughness_from_normal(image: TYPE_IMAGE) -> TYPE_IMAGE: + """Roughness from a normal map.""" + up_vector = np.array([0, 0, 1]) + image = 1 - np.dot(image, up_vector) + image = (image - image.min()) / (image.max() - image.min()) + image = (255 * image).astype(np.uint8) + return image_grayscale(image) + +def roughness_from_albedo(image: TYPE_IMAGE) -> TYPE_IMAGE: + """Roughness from an albedo map.""" + kernel_size = 3 + image = cv2.Laplacian(image, cv2.CV_64F, ksize=kernel_size) + image = (image - image.min()) / (image.max() - image.min()) + image = (255 * image).astype(np.uint8) + return image_grayscale(image) + +def roughness_from_albedo_normal(albedo: TYPE_IMAGE, normal: TYPE_IMAGE, blur:int=2, blend:float=0.5, iterations:int=3) -> TYPE_IMAGE: + normal = roughness_from_normal(normal) + normal = image_normalize(normal) + albedo = roughness_from_albedo(albedo) + albedo = image_normalize(albedo) + rough = image_lerp(normal, albedo, alpha=blend) + rough = image_normalize(rough) + image = image_lerp(normal, rough, alpha=blend) + iterations = min(16, max(2, iterations)) + blur += (blur % 2 == 0) + step = 1 / 2 ** iterations + for i in range(iterations): + image = cv2.add(normal * step, image * step) + image = cv2.GaussianBlur(image, (blur + i * 2, blur + i * 2), 3 * i) + + inverted = 255 - image_normalize(image) + inverted = cv2.subtract(inverted, albedo) * 0.5 + inverted = cv2.GaussianBlur(inverted, (blur, blur), blur) + inverted = image_normalize(inverted) + + image = cv2.add(image * 0.5, inverted * 0.5) + for i in range(iterations): + image = cv2.GaussianBlur(image, (blur, blur), blur) + + image = cv2.add(image * 0.5, inverted * 0.5) + for i in range(iterations): + image = cv2.GaussianBlur(image, (blur, blur), blur) + + image = image_normalize(image) + return image diff --git a/sup/lexicon.py b/sup/lexicon.py index 5014aa0..d6daf2e 100644 --- a/sup/lexicon.py +++ b/sup/lexicon.py @@ -106,15 +106,12 @@ class Lexicon(metaclass=LexiconMeta): FONT_SIZE = 'SIZE', "Text Size" FORMAT = 'FORMAT', "Format" FPS = '🏎ī¸', "Frames per second" - FRAGMENT = 'FRAGMENT', "Shader Fragment Program" FRAME = '⏚ī¸', "Frame" FREQ = 'FREQ', "Frequency" FUNC = '⚒ī¸', "Function" G = '🟩', "Green" GAMMA = '🔆', "Gamma" GI = '💚', "Green Channel" - GLSL_CHANNEL = 'iChannel', "GL Image Input" - GLSL_VAR = 'iVar', "GL Variable Input" GRADIENT = '🇲đŸ‡ē', "Gradient" H = '🇭', "Hue" HI = 'HI', "High / Top of range" @@ -174,6 +171,8 @@ class Lexicon(metaclass=LexiconMeta): PIXEL_B = '👾B', "Pixel Data (RGBA, RGB or Grayscale)" PREFIX = 'PREFIX', "Prefix" PRESET = 'PRESET', "Preset" + PROG_VERT = 'VERTEX', "Select a vertex program to load" + PROG_FRAG = 'FRAGMENT', "Select a fragment program to load" PROJECTION = 'PROJ', "Projection" QUALITY = 'QUALITY', "Quality" QUALITY_M = 'MOTION', "Motion Quality" diff --git a/sup/shader.py b/sup/shader.py index 0f3cea4..1567672 100644 --- a/sup/shader.py +++ b/sup/shader.py @@ -8,13 +8,15 @@ import re from typing import Tuple +import cv2 import glfw import numpy as np import OpenGL.GL as gl from loguru import logger + +from Jovimetrix import load_file from Jovimetrix.sup.util import EnumConvertType, parse_value -from Jovimetrix.sup.image import image_convert # ============================================================================= @@ -75,26 +77,16 @@ class GLSLShader(): } """ - PROG_FRAGMENT = """ -uniform sampler2D imageA; -uniform sampler2D imageB; -uniform float size; // 7. -uniform float time_scale; // 19. + PROG_FRAGMENT = """uniform sampler2D imageA; void mainImage( out vec4 fragColor, vec2 fragCoord ) { - vec2 u = floor(fragCoord / size); - float s = dot(u, u) + iTime / time_scale; - u = mod(u, 2.); - vec2 uv = fragCoord.xy / iResolution.xy; - vec3 col2 = texture2D(imageB, uv).rgb; vec3 col = texture2D(imageA, uv).rgb; - fragColor = vec4(col + (floor(sin(s)) - u.y - floor(cos(s)) - u.xxxx).rgb, 1.0); + fragColor = vec4(col, 1.0); } """ - PROG_VERTEX = """ -#version 330 core + PROG_VERTEX = """#version 330 core void main() { vec2 verts[3] = vec2[](vec2(-1, -1), vec2(3, -1), vec2(-1, 3)); @@ -113,7 +105,6 @@ def __init__(self, vertex:str=None, fragment:str=None, width:int=IMAGE_SIZE_DEFA self.__size_changed = False self.__size: Tuple[int, int] = (max(width, IMAGE_SIZE_MIN), max(height, IMAGE_SIZE_MIN)) - self.__program = None self.__source_vertex: None self.__source_fragment: None @@ -131,6 +122,7 @@ def __init__(self, vertex:str=None, fragment:str=None, width:int=IMAGE_SIZE_DEFA self.program(vertex, fragment) def __compile_shader(self, source:str, shader_type:str) -> None: + glfw.make_context_current(self.__window) shader = gl.glCreateShader(shader_type) gl.glShaderSource(shader, source) gl.glCompileShader(shader) @@ -139,8 +131,18 @@ def __compile_shader(self, source:str, shader_type:str) -> None: logger.debug(f"{shader_type} compiled") return shader + def __init_window(self) -> None: + self.__cleanup() + glfw.window_hint(glfw.VISIBLE, glfw.FALSE) + self.__window = glfw.create_window(self.__size[0], self.__size[1], "hidden", None, None) + if not self.__window: + raise RuntimeError("GLFW did not init window") + self.__framebuffer() + self.program() + def __framebuffer(self) -> None: # match the window to the buffer size... + glfw.make_context_current(self.__window) glfw.set_window_size(self.__window, self.__size[0], self.__size[1]) if self.__fbo is None: self.__fbo = gl.glGenFramebuffers(1) @@ -160,18 +162,29 @@ def __framebuffer(self) -> None: raise RuntimeError("Framebuffer is not complete") # dump all the old texture slots? - old = [v[3] for v in self.__userVar.values() if v[0] == 'sampler2D'] - gl.glDeleteTextures(old) + # old = [v[3] for v in self.__userVar.values() if v[0] == 'sampler2D'] + # gl.glDeleteTextures(old) gl.glViewport(0, 0, self.__size[0], self.__size[1]) + def __cleanup(self) -> None: + glfw.make_context_current(self.__window) + old = [v[3] for v in self.__userVar.values() if v[0] == 'sampler2D'] + if len(old): + gl.glDeleteTextures(old) + if self.__fbo_texture: + gl.glDeleteTextures(1, [self.__fbo_texture]) + self.__fbo_texture = None + if self.__fbo: + gl.glDeleteFramebuffers(1, [self.__fbo]) + self.__fbo = None + if self.__window: + glfw.destroy_window(self.__window) + self.__window = None + def __del__(self) -> None: + glfw.make_context_current(self.__window) if gl: - if self.__fbo_texture: - gl.glDeleteTextures(1, [self.__fbo_texture]) - if self.__fbo: - gl.glDeleteFramebuffers(1, [self.__fbo]) - if self.__window: - glfw.destroy_window(self.__window) + self.__cleanup() glfw.terminate() @property @@ -244,6 +257,17 @@ def bgcolor(self) -> Tuple[int, ...]: def bgcolor(self, color:Tuple[int, ...]) -> None: self.__bgcolor = tuple(x / 255. for x in color) + def program_load(self, vertex_file:str=None, frag_file:str=None) -> None: + """Loads external file source as Vertex and/or Fragment programs.""" + vertex = None + if vertex_file is not None: + vertex = load_file(vertex_file) + + fragment = None + if frag_file is not None: + fragment = load_file(frag_file) + self.program(vertex, fragment) + def program(self, vertex:str=None, fragment:str=None) -> None: if (vertex := self.__source_vertex_raw if vertex is None else vertex) is None: logger.debug("Vertex program is empty. Using Default.") @@ -254,10 +278,12 @@ def program(self, vertex:str=None, fragment:str=None) -> None: fragment = self.PROG_FRAGMENT if vertex != self.__source_vertex_raw or fragment != self.__source_fragment_raw: - # glfw.make_context_current(self.__window) - + glfw.make_context_current(self.__window) if self.__program: - gl.glDeleteProgram(self.__program) + try: + gl.glDeleteProgram(self.__program) + except Exception as e: + logger.warning(e) self.__source_vertex = self.__compile_shader(vertex, gl.GL_VERTEX_SHADER) fragment_full = self.PROG_HEADER + fragment + self.PROG_FOOTER @@ -282,7 +308,6 @@ def program(self, vertex:str=None, fragment:str=None) -> None: 'iMouse': gl.glGetUniformLocation(self.__program, "iMouse") } - texture_count = 0 self.__userVar = {} # read the fragment and setup the vars.... for match in RE_VARIABLE.finditer(fragment): @@ -302,7 +327,12 @@ def program(self, vertex:str=None, fragment:str=None) -> None: tex_loc ] + logger.info("program changed") + self.render() + self.render() + def render(self, time_delta:float=0., **kw) -> np.ndarray: + glfw.make_context_current(self.__window) self.runtime = time_delta gl.glUseProgram(self.__program) @@ -333,6 +363,7 @@ def render(self, time_delta:float=0., **kw) -> np.ndarray: op = gl.GL_RGBA if val.shape[2] == 4 else gl.GL_RGB val = val[::-1,:] val = val.astype(np.float32) / 255.0 + val = cv2.resize(val, self.__size, interpolation=cv2.INTER_LINEAR) gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, op, val.shape[1], val.shape[0], 0, op, gl.GL_FLOAT, val) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) diff --git a/sup/util.py b/sup/util.py index 3d9d616..7e0333a 100644 --- a/sup/util.py +++ b/sup/util.py @@ -54,6 +54,70 @@ class EnumSwizzle(Enum): # === SUPPORT === # ============================================================================= +def deep_merge_dict(*dicts: dict) -> dict: + """ + Deep merge multiple dictionaries recursively. + + Args: + *dicts: Variable number of dictionaries to be merged. + + Returns: + dict: Merged dictionary. + """ + def _deep_merge(d1: Any, d2: Any) -> Any: + if not isinstance(d1, dict) or not isinstance(d2, dict): + return d2 + + merged_dict = d1.copy() + + for key in d2: + if key in merged_dict: + if isinstance(merged_dict[key], dict) and isinstance(d2[key], dict): + merged_dict[key] = _deep_merge(merged_dict[key], d2[key]) + elif isinstance(merged_dict[key], list) and isinstance(d2[key], list): + merged_dict[key].extend(d2[key]) + else: + merged_dict[key] = d2[key] + else: + merged_dict[key] = d2[key] + return merged_dict + + merged = {} + for d in dicts: + merged = _deep_merge(merged, d) + return merged + +def grid_make(data: List[Any]) -> Tuple[List[List[Any]], int, int]: + """ + Create a 2D grid from a 1D list. + + Args: + data (List[Any]): Input data. + + Returns: + Tuple[List[List[Any]], int, int]: A tuple containing the 2D grid, number of columns, + and number of rows. + """ + size = len(data) + grid = int(math.sqrt(size)) + if grid * grid < size: + grid += 1 + if grid < 1: + return [], 0, 0 + + rows = size // grid + if size % grid != 0: + rows += 1 + + ret = [] + cols = 0 + for j in range(rows): + end = min((j + 1) * grid, len(data)) + cols = max(cols, end - j * grid) + d = [data[i] for i in range(j * grid, end)] + ret.append(d) + return ret, cols, rows + def parse_dynamic(data:dict, prefix:str, typ:EnumConvertType, default: Any) -> List[Any]: """Convert iterated input field(s) based on a s into a single compound list of entries. @@ -248,6 +312,33 @@ def parse_param(data:dict, key:str, typ:EnumConvertType, default: Any, val = [val] return [parse_value(v, typ, default, clip_min, clip_max, zero) for v in val] +def path_next(pattern: str) -> str: + """ + Finds the next free path in an sequentially named list of files + """ + i = 1 + while os.path.exists(pattern % i): + i = i * 2 + + a, b = (i // 2, i) + while a + 1 < b: + c = (a + b) // 2 + a, b = (c, b) if os.path.exists(pattern % c) else (a, c) + return pattern % b + +def update_nested_dict(d, path, value) -> None: + keys = path.split('.') + current = d + for key in keys[:-1]: + current = current.setdefault(key, {}) + last_key = keys[-1] + + # Check if the key already exists + if last_key in current and isinstance(current[last_key], dict): + current[last_key].update(value) + else: + current[last_key] = value + def vector_swap(pA: Any, pB: Any, swap_x: EnumSwizzle, x:float, swap_y:EnumSwizzle, y:float, swap_z:EnumSwizzle, z:float, swap_w:EnumSwizzle, w:float) -> List[float]: """Swap out a vector's values with another vector's values, or a constant fill.""" @@ -266,19 +357,6 @@ def parse(target, targetB, swap, val) -> float: parse(pA, pB, swap_w, w) ] -def update_nested_dict(d, path, value) -> None: - keys = path.split('.') - current = d - for key in keys[:-1]: - current = current.setdefault(key, {}) - last_key = keys[-1] - - # Check if the key already exists - if last_key in current and isinstance(current[last_key], dict): - current[last_key].update(value) - else: - current[last_key] = value - def zip_longest_fill(*iterables: Any) -> Generator[Tuple[Any, ...], None, None]: """ Zip longest with fill value. @@ -310,81 +388,3 @@ def zip_longest_fill(*iterables: Any) -> Generator[Tuple[Any, ...], None, None]: values[i] = current_value yield tuple(values) - -def deep_merge_dict(*dicts: dict) -> dict: - """ - Deep merge multiple dictionaries recursively. - - Args: - *dicts: Variable number of dictionaries to be merged. - - Returns: - dict: Merged dictionary. - """ - def _deep_merge(d1: Any, d2: Any) -> Any: - if not isinstance(d1, dict) or not isinstance(d2, dict): - return d2 - - merged_dict = d1.copy() - - for key in d2: - if key in merged_dict: - if isinstance(merged_dict[key], dict) and isinstance(d2[key], dict): - merged_dict[key] = _deep_merge(merged_dict[key], d2[key]) - elif isinstance(merged_dict[key], list) and isinstance(d2[key], list): - merged_dict[key].extend(d2[key]) - else: - merged_dict[key] = d2[key] - else: - merged_dict[key] = d2[key] - return merged_dict - - merged = {} - for d in dicts: - merged = _deep_merge(merged, d) - return merged - -def grid_make(data: List[Any]) -> Tuple[List[List[Any]], int, int]: - """ - Create a 2D grid from a 1D list. - - Args: - data (List[Any]): Input data. - - Returns: - Tuple[List[List[Any]], int, int]: A tuple containing the 2D grid, number of columns, - and number of rows. - """ - size = len(data) - grid = int(math.sqrt(size)) - if grid * grid < size: - grid += 1 - if grid < 1: - return [], 0, 0 - - rows = size // grid - if size % grid != 0: - rows += 1 - - ret = [] - cols = 0 - for j in range(rows): - end = min((j + 1) * grid, len(data)) - cols = max(cols, end - j * grid) - d = [data[i] for i in range(j * grid, end)] - ret.append(d) - return ret, cols, rows - -def path_next(pattern: str) -> str: - """ - Finds the next free path in an sequentially named list of files - """ - i = 1 - while os.path.exists(pattern % i): - i = i * 2 - - a, b = (i // 2, i) - while a + 1 < b: - c = (a + b) // 2 - a, b = (c, b) if os.path.exists(pattern % c) else (a, c) - return pattern % b diff --git a/web/core/core_cozy_menu.js b/web/core/core_cozy_menu.js index 81374df..af28ce2 100644 --- a/web/core/core_cozy_menu.js +++ b/web/core/core_cozy_menu.js @@ -9,29 +9,29 @@ import { CONVERTED_TYPE, convertToInput } from '../util/util_widget.js' app.registerExtension({ name: "jovimetrix.cozy.menu", async beforeRegisterNodeDef(nodeType, nodeData, app) { - if (nodeData.name.includes("(JOV)")) { + if (!nodeData.name.includes("(JOV)")) { return; } + let matchingTypes = []; const inputTypes = nodeData.input; - if (!inputTypes) { - return; - } - - const matchingTypes = ['required', 'optional'] - .flatMap(type => Object.entries(inputTypes[type] || []) - ); - if (matchingTypes.length == 0) { - return; + if (inputTypes) { + matchingTypes = ['required', 'optional'] + .flatMap(type => Object.entries(inputTypes[type] || []) + ); + if (matchingTypes.length == 0) { + return; + } } // MENU CONVERSIONS const getExtraMenuOptions = nodeType.prototype.getExtraMenuOptions; - nodeType.prototype.getExtraMenuOptions = function (_, options) { + nodeType.prototype.getExtraMenuOptions = async function (_, options) { const me = getExtraMenuOptions?.apply(this, arguments); if (this.widgets === undefined) { return me; } + const convertToInputArray = []; const widgets = Object.values(this.widgets); for (const [widgetName, widgetType] of matchingTypes) { @@ -39,14 +39,20 @@ app.registerExtension({ if (widget && widget.type !== CONVERTED_TYPE && (widget.options?.forceInput === undefined || widget.options?.forceInput === false) && widget.options?.menu !== false) { - const convertToInputObject = { - content: `Convert ${widget.name} to input`, + content: `Convsert ${widget.name} to input`, callback: () => convertToInput(this, widget, widgetType) }; convertToInputArray.push(convertToInputObject); } } + // remove all the options that start with the word "Convert" from the options... + if (options) { + options = options.filter(option => { + return typeof option?.content !== 'string' || !option?.content.startsWith('Convert'); + }); + } + if (convertToInputArray.length) { options.push(...convertToInputArray, null); } diff --git a/web/nodes/delay.js b/web/nodes/delay.js index 3a5d124..8f89516 100644 --- a/web/nodes/delay.js +++ b/web/nodes/delay.js @@ -11,6 +11,8 @@ import { api_post } from '../util/util_api.js' import { bubbles } from '../util/util_fun.js' const _id = "DELAY (JOV) ✋đŸŊ" +const EVENT_JOVI_DELAY = "jovi-delay-user"; +const EVENT_JOVI_UPDATE = "jovi-delay-update"; app.registerExtension({ name: 'jovimetrix.node.' + _id, @@ -64,11 +66,17 @@ app.registerExtension({ window.bubbles_alive = false; // app.canvas.setDirty(true); } - api.addEventListener("jovi-delay-user", python_delay_user); async function python_delay_update(event) { } - api.addEventListener("jovi-delay-update", python_delay_update); + + api.addEventListener(EVENT_JOVI_DELAY, python_delay_user); + api.addEventListener(EVENT_JOVI_UPDATE, python_delay_update); + + this.onDestroy = () => { + api.removeEventListener(EVENT_JOVI_DELAY, python_glsl_error); + api.removeEventListener(EVENT_JOVI_UPDATE, python_delay_update); + }; return me; } diff --git a/web/nodes/glsl.js b/web/nodes/glsl.js index 5e41ca5..1ebb71a 100644 --- a/web/nodes/glsl.js +++ b/web/nodes/glsl.js @@ -8,12 +8,13 @@ import { api } from "../../../scripts/api.js"; import { app } from "../../../scripts/app.js"; import { fitHeight } from '../util/util.js' import { widget_hide, widget_show } from '../util/util_widget.js'; -import { api_cmd_jovian } from '../util/util_api.js'; +import { api_post, api_cmd_jovian } from '../util/util_api.js'; import { flashBackgroundColor } from '../util/util_fun.js'; const _id = "GLSL (JOV) 🍩"; const EVENT_JOVI_GLSL_ERROR = "jovi-glsl-error"; const EVENT_JOVI_GLSL_TIME = "jovi-glsl-time"; +const EVENT_JOVI_GLSL_REGISTER = "jovi-register-glsl"; const RE_VARIABLE = /uniform\s*(\w*)\s*(\w*);(?:.*\/{2}\s*([A-Za-z0-9\-\.,\s]+)){0,1}\s*$/gm app.registerExtension({ @@ -28,14 +29,15 @@ app.registerExtension({ const me = onNodeCreated?.apply(this); const self = this; const widget_time = this.widgets.find(w => w.name === '🕛'); - widget_time.options.menu = false; const widget_batch = this.widgets.find(w => w.name === 'BATCH'); - widget_batch.options.menu = false; + // const widget_wait = this.widgets.find(w => w.name === '✋đŸŊ'); - widget_wait.options.menu = false; const widget_reset = this.widgets.find(w => w.name === 'RESET'); - widget_reset.options.menu = false; + const widget_vertex = this.widgets.find(w => w.name === 'VERTEX'); const widget_fragment = this.widgets.find(w => w.name === 'FRAGMENT'); + widget_wait.options.menu = false; + widget_reset.options.menu = false; + widget_vertex.options.menu = false; widget_fragment.options.menu = false; let widget_param = this.inputs?.find(w => w.name === 'PARAM'); if (widget_param === undefined) { @@ -93,7 +95,6 @@ app.registerExtension({ widgets.push(varName); }); - while (self.inputs?.length > widgets.length) { let idx = 0; self.inputs.forEach(i => { @@ -110,6 +111,10 @@ app.registerExtension({ shader_changed(); }); + widget_vertex.inputEl.addEventListener('input', function () { + shader_changed(); + }); + widget_batch.callback = () => { widget_hide(self, widget_reset, '-jov'); widget_hide(self, widget_wait, '-jov'); diff --git a/web/nodes/queue.js b/web/nodes/queue.js index 9b749db..f47a681 100644 --- a/web/nodes/queue.js +++ b/web/nodes/queue.js @@ -12,8 +12,10 @@ import { flashBackgroundColor } from '../util/util_fun.js' import { fitHeight, TypeSlotEvent, TypeSlot } from '../util/util.js' import { widget_hide, widget_show } from '../util/util_widget.js' -const _id = "QUEUE (JOV) 🗃" -const _prefix = 'đŸĻ„' +const _id = "QUEUE (JOV) 🗃"; +const _prefix = 'đŸĻ„'; +const EVENT_JOVI_PING = "jovi-queue-ping"; +const EVENT_JOVI_DONE = "jovi-queue-done"; app.registerExtension({ name: 'jovimetrix.node.' + _id, @@ -96,8 +98,14 @@ app.registerExtension({ await flashBackgroundColor(self.widget_queue.inputEl, 650, 4, "#995242CC"); } - api.addEventListener("jovi-queue-ping", python_queue_ping); - api.addEventListener("jovi-queue-done", python_queue_done); + api.addEventListener(EVENT_JOVI_PING, python_queue_ping); + api.addEventListener(EVENT_JOVI_DONE, python_queue_done); + + this.onDestroy = () => { + api.removeEventListener(EVENT_JOVI_PING, python_queue_ping); + api.removeEventListener(EVENT_JOVI_DONE, python_queue_done); + }; + setTimeout(() => { widget_value.callback(); }, 10); return me; } diff --git a/web/nodes/tick.js b/web/nodes/tick.js index b885a57..3e19306 100644 --- a/web/nodes/tick.js +++ b/web/nodes/tick.js @@ -8,7 +8,8 @@ import { api } from "../../../scripts/api.js"; import { app } from "../../../scripts/app.js" import { api_cmd_jovian } from '../util/util_api.js' -const _id = "TICK (JOV) ⏱" +const _id = "TICK (JOV) ⏱"; +const EVENT_JOVI_TICK = "jovi-tick"; app.registerExtension({ name: 'jovimetrix.node.' + _id, @@ -33,7 +34,11 @@ app.registerExtension({ } self.widget_count.value = event.detail.i; } + api.addEventListener("jovi-tick", python_tick); + this.onDestroy = () => { + api.removeEventListener(EVENT_JOVI_TICK, python_tick); + }; return me; } } diff --git a/web/util/util_api.js b/web/util/util_api.js index 54810b1..0fda35d 100644 --- a/web/util/util_api.js +++ b/web/util/util_api.js @@ -33,3 +33,17 @@ export async function api_cmd_jovian(id, cmd) { }), }) } + +export async function api_cmd_glsl_vertex(id, vertex, fragment) { + return api.fetchApi('/jovimetrix/glsl', { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: id, + vertex: vertex, + fragment: fragment + }), + }) +} diff --git a/web/widget/widget_vector.js b/web/widget/widget_vector.js index d7f3d1a..63bc511 100644 --- a/web/widget/widget_vector.js +++ b/web/widget/widget_vector.js @@ -244,7 +244,7 @@ app.registerExtension({ if (myTypes.includes(widget.type)) { const who = matchingTypes.find(w => w[0] === widget.name) const convertToInputObject = { - content: `Convert ${widget.name} to input`, + content: `Convert vector ${widget.name} to input`, callback: () => convertToInput(this, widget, who[1]) }; convertToInputArray.push(convertToInputObject);