diff --git a/playwright/live-painting.test.ts b/playwright/live-painting.test.ts new file mode 100644 index 000000000..25d11fc74 --- /dev/null +++ b/playwright/live-painting.test.ts @@ -0,0 +1,24 @@ +import type { ElectronApplication, Page } from "@playwright/test"; +import { test, expect } from "@playwright/test"; +import { _electron as electron } from "playwright"; + +let electronApp: ElectronApplication; +let page: Page; + +test.beforeAll(async () => { + electronApp = await electron.launch({ args: ["."] }); + const isPackaged = await electronApp.evaluate(async ({ app }) => app.isPackaged); + + expect(isPackaged).toBe(false); +}); + +test.afterAll(async () => { + await electronApp.close(); +}); + +test.skip("Open Live Painting", async () => { + page = await electronApp.firstWindow(); + + await page.getByTestId("sidebar-live-painting").click(); + await expect(page.url()).toContain("live-painting"); +}); diff --git a/resources/python/live-painting/main.py b/resources/python/live-painting/main.py index 990ea3e98..989ca928f 100644 --- a/resources/python/live-painting/main.py +++ b/resources/python/live-painting/main.py @@ -1,14 +1,41 @@ -from diffusers import ( - StableDiffusionImg2ImgPipeline, - AutoencoderTiny, - EulerAncestralDiscreteScheduler, -) +import io +import sys +from contextlib import contextmanager +import json + +print(json.dumps({"status": "starting"})) + + +@contextmanager +def suppress_print(debug=False): + # Hide messages printed to stdout / stderr + + if not debug: + original_stdout = sys.stdout + original_stderr = sys.stderr + sys.stdout = io.StringIO() + sys.stderr = io.StringIO() + + try: + yield + finally: + sys.stdout = original_stdout + sys.stderr = original_stderr + else: + yield + + +# Get rid of message, that Triton is not available +with suppress_print(): + from diffusers import ( + StableDiffusionImg2ImgPipeline, + AutoencoderTiny, + EulerAncestralDiscreteScheduler, + ) import torch import time from PIL import Image import queue -import sys -import json import threading import os import time @@ -16,15 +43,74 @@ import torch from diffusers.utils import load_image from sfast.compilers.diffusion_pipeline_compiler import compile, CompilationConfig +import argparse -def read_stdin_loop(): +# Torch optimizations +torch.set_grad_enabled(False) +torch.backends.cuda.matmul.allow_tf32 = True +torch.backends.cudnn.allow_tf32 = True + + +def parse_args(): + parser = argparse.ArgumentParser(description="Control model and paths via CLI.") + parser.add_argument( + "--model_path", + type=str, + help="Path to the model directory or identifier.", + required=True, + ) + parser.add_argument( + "--vae_path", + type=str, + help="Path to the vae directory or identifier.", + required=True, + ) + parser.add_argument( + "--input_image_path", + type=str, + default="live-canvas-frontend-user-data.png", + help="Path to the input image file.", + required=True, + ) + parser.add_argument( + "--output_image_path", + type=str, + default="live-canvas-generate-image-output.png", + help="Path for the output image file.", + ) + parser.add_argument( + "--disable_stablefast", + action="store_true", + help="Disable the stablefast compilation for faster loading during development.", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug output.", + ) + + args = parser.parse_args() + return args + + +def read_stdin_loop(params_queue, shutdown_event): for line in sys.stdin: if line.strip(): try: - params_queue.put(json.loads(line)) + data = json.loads(line) + + if data.get("command") == "shutdown": + shutdown_event.set() + break + else: + params_queue.put(data) except json.JSONDecodeError as e: - print(f"Error decoding JSON: {e}") + print( + json.dumps( + {"status": "error", "message": f"Error decoding JSON: {e}"} + ) + ) def safe_rename_with_retries(src, dst, max_retries=5, delay=0.005): @@ -68,134 +154,126 @@ def calculate_min_inference_steps(strength): return math.ceil(1.0 / strength) -# Initial/default values for parameters -prompt = None -seed = None -strength = None -guidance_scale = None -input_path = "live-canvas-frontend-user-data.png" -output_path = "live-canvas-generate-image-output.png" +def prepare_pipeline(model_path, vae_path, disable_stablefast=False): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + model_path, + torch_dtype=torch.float16, + variant="fp16", + safety_checker=None, + requires_safety_checker=False, + ) -# Queue to hold parameters received from stdin -params_queue = queue.Queue() + pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.safety_checker = None + pipe.to(torch.device("cuda")) -# Start the stdin reading thread -stdin_thread = threading.Thread(target=read_stdin_loop, daemon=True) -stdin_thread.start() + # Load the stable-fast default config + config = CompilationConfig.Default() + # Whether to preserve parameters when freezing the model. + # If True, parameters will be preserved, but the model will be a bit slower. + # If False, parameters will be marked as constants, and the model will be faster. + config.preserve_parameters = False -# Torch optimizations -torch.set_grad_enabled(False) -torch.backends.cuda.matmul.allow_tf32 = True -torch.backends.cudnn.allow_tf32 = True - + # xformers and Triton are suggested for achieving best performance. + try: + import xformers -pipe = StableDiffusionImg2ImgPipeline.from_pretrained( - "stabilityai/sd-turbo", - torch_dtype=torch.float16, - variant="fp16", - safety_checker=None, - requires_safety_checker=False, -) - -pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) -pipe.safety_checker = None -pipe.to(torch.device("cuda")) - -# Load the stable-fast default config -config = CompilationConfig.Default() - -# Whether to preserve parameters when freezing the model. -# If True, parameters will be preserved, but the model will be a bit slower. -# If False, parameters will be marked as constants, and the model will be faster. -config.preserve_parameters = False - -# xformers and Triton are suggested for achieving best performance. -try: - import xformers - - config.enable_xformers = True -except ImportError: - print("xformers not installed, skip") -try: - import triton - - config.enable_triton = True -except ImportError: - print("Triton not installed, skip") - -# CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead. -# But it can increase the amount of GPU memory used. -config.enable_cuda_graph = True - -pipe.width = 512 -pipe.height = 512 - -pipe.vae = AutoencoderTiny.from_pretrained("madebyollin/taesd").to( - device=pipe.device, dtype=pipe.dtype -) - -# Channels-last memory format -# see https://huggingface.co/docs/diffusers/optimization/memory#channelslast-memory-format -pipe.unet.to(memory_format=torch.channels_last) -pipe.vae.to(memory_format=torch.channels_last) - -# Disable inference progress bar -pipe.set_progress_bar_config(leave=False) -pipe.set_progress_bar_config(disable=True) - -pipe = compile(pipe, config) - -# Warmup -print("warmup started") -for _ in range(10): - init_image = load_image_with_retry(input_path) - output_image = pipe( - prompt="the moon, 4k", - image=init_image, - height=512, - width=512, - num_inference_steps=1, - num_images_per_prompt=1, - strength=1.0, - guidance_scale=0.0, - ).images[0] -print("warmup done") - - -def main(): - global prompt, seed, strength, guidance_scale, input_path, output_path - - while True: + config.enable_xformers = True + except ImportError: + print("xformers not installed, skip") + try: + import triton + + config.enable_triton = True + except ImportError: + print("Triton not installed, skip") + + # CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead. + # But it can increase the amount of GPU memory used. + config.enable_cuda_graph = True + + pipe.width = 512 + pipe.height = 512 + + pipe.vae = AutoencoderTiny.from_pretrained(vae_path).to( + device=pipe.device, dtype=pipe.dtype + ) + + # Channels-last memory format + # see https://huggingface.co/docs/diffusers/optimization/memory#channelslast-memory-format + pipe.unet.to(memory_format=torch.channels_last) + pipe.vae.to(memory_format=torch.channels_last) + + # Disable inference progress bar + pipe.set_progress_bar_config(leave=False) + pipe.set_progress_bar_config(disable=True) + + # Disable stable fast for faster startup of the script, but slower inference + if not disable_stablefast: + pipe = compile(pipe, config) + + return pipe + + +def warmup(pipe, input_image_path): + # Warmup + for _ in range(6): + init_image = load_image_with_retry(input_image_path) + pipe( + prompt="the moon, 4k", + image=init_image, + height=512, + width=512, + num_inference_steps=1, + num_images_per_prompt=1, + strength=1.0, + guidance_scale=0.0, + ).images[0] + + +def main(pipe, input_image_path, output_image_path, shutdown_event): + # Initial/default values for parameters + prompt = "realistic face, 4k" + seed = 1 + strength = 0.99 + guidance_scale = None + + # Queue to hold parameters received from stdin + params_queue = queue.Queue() + + # Read from stdin + stdin_thread = threading.Thread( + target=read_stdin_loop, args=(params_queue, shutdown_event), daemon=True + ) + stdin_thread.start() + + while not shutdown_event.is_set(): try: # Update parameters if new ones are available while not params_queue.empty(): parameters = params_queue.get_nowait() prompt = parameters.get("prompt", prompt) seed = parameters.get("seed", seed) - input_path = parameters.get("input_path", input_path) strength = parameters.get("strength", strength) guidance_scale = parameters.get("guidance_scale", guidance_scale) - output_path = parameters.get("output_path", output_path) print(f"Updated parameters {parameters}") except queue.Empty: pass # No new parameters, proceed with the existing ones # Only generate an image if the prompt is not empty if prompt is not None and prompt.strip(): - start_time = time.time() - torch.manual_seed(seed) - init_image = load_image_with_retry(input_path) + init_image = load_image_with_retry(input_image_path) # Image couldn't be loaded, skip this iteration if init_image is None: continue strength_ = float(strength) - guidance_scale_ = float(guidance_scale) - denoise_steps_ = calculate_min_inference_steps(strength_) + # guidance_scale_ = float(guidance_scale) + # denoise_steps_ = calculate_min_inference_steps(strength_) image = pipe( prompt, @@ -208,35 +286,39 @@ def main(): guidance_scale=1.5, ).images[0] - end_time = time.time() - # Save file - image.save(f"{output_path}.tmp.png") - safe_rename_with_retries(f"{output_path}.tmp.png", output_path) + image.save(f"{output_image_path}.tmp.png") + safe_rename_with_retries(f"{output_image_path}.tmp.png", output_image_path) + + print(json.dumps({"status": "image_generated"})) - duration = (end_time - start_time) * 1000 - # print(f"{duration:.2f} ms") - times.append(duration) else: image = Image.new("RGB", (512, 512), color="white") - image.save(f"{output_path}.tmp.png") - safe_rename_with_retries(f"{output_path}.tmp.png", output_path) + image.save(f"{output_image_path}.tmp.png") + safe_rename_with_retries(f"{output_image_path}.tmp.png", output_image_path) if __name__ == "__main__": - times = [] + # Used for gracefully shutting down the script + shutdown_event = threading.Event() try: - print("started loop") - main() + args = parse_args() + pipe = None + + with suppress_print(args.debug): + pipe = prepare_pipeline( + args.model_path, args.vae_path, args.disable_stablefast + ) + warmup(pipe, args.input_image_path) + + print(json.dumps({"status": "started"})) + + main(pipe, args.input_image_path, args.output_image_path, shutdown_event) + + print(json.dumps({"status": "shutdown"})) + except Exception as error: - print(error) + print(json.dumps({"status": "error", "message": str(error)})) except: - if times: - total_time = sum(times) - average_time = total_time / len(times) - print( - f"Total time: {total_time:.2f} ms, Total executions: {len(times)}, Average time: {average_time:.2f} ms" - ) - else: - print(f"No operations were performed or exception") + print(json.dumps({"status": "stopped"})) diff --git a/resources/python/live-painting/test-resources/input.png b/resources/python/live-painting/test-resources/input.png new file mode 100644 index 000000000..5f52c3a2e Binary files /dev/null and b/resources/python/live-painting/test-resources/input.png differ diff --git a/src/client/organisms/layout/index.tsx b/src/client/organisms/layout/index.tsx index 151a16dcc..ac2920ba9 100644 --- a/src/client/organisms/layout/index.tsx +++ b/src/client/organisms/layout/index.tsx @@ -50,8 +50,12 @@ export function Layout({ children }: { children?: ReactNode }) { }> {t("common:training")} - }> - {t("common:livePainting")} + } + data-testid="sidebar-live-painting" + > + {t("labels:livePainting")} (null); + const context = useRef(null); + const isDrawing = useRef(false); + const [, setImage] = useAtom(imageAtom); + + function startDrawing(event: ReactPointerEvent) { + if (!canvas.current) { + return; + } + + isDrawing.current = true; + const rect = canvas.current.getBoundingClientRect(); + context.current?.beginPath(); + context.current?.moveTo(event.clientX - rect.left, event.clientY - rect.top); + } + + useEffect(() => { + const canvasElement = canvas.current; + let animationFame: number; + if (!canvasElement) { + return; + } + + canvasElement.height = 512; + canvasElement.width = 512; + context.current = canvasElement.getContext("2d"); + + function draw(event: MouseEvent) { + if (!isDrawing.current || !canvas.current) { + return; + } + + const rect = canvas.current.getBoundingClientRect(); + context.current?.lineTo(event.clientX - rect.left, event.clientY - rect.top); + context.current?.stroke(); + const dataUrl = canvas.current.toDataURL(); + setImage(dataUrl); + window.ipc.send(buildKey([ID.LIVE_PAINT], { suffix: ":dataUrl" }), dataUrl); + } + + function handleMouseUp() { + if (isDrawing.current) { + context.current?.closePath(); + isDrawing.current = false; + } + + cancelAnimationFrame(animationFame); + } + + function handleMouseMove(event: MouseEvent) { + animationFame = requestAnimationFrame(() => { + draw(event); + }); + } + + if (context.current) { + context.current.fillStyle = "#ffffff"; + context.current.rect(0, 0, canvasElement.width, canvasElement.height); + context.current.fill(); + context.current.strokeStyle = "#000000"; + context.current.lineWidth = 5; + context.current.lineJoin = "round"; + context.current.lineCap = "round"; + + document.addEventListener("mousemove", handleMouseMove, { passive: true }); + document.addEventListener("mouseup", handleMouseUp); + } + + return () => { + document.removeEventListener("mousemove", handleMouseMove); + document.removeEventListener("mouseup", handleMouseUp); + cancelAnimationFrame(animationFame); + }; + }, [setImage]); + + return ( + + + + ); +} + +function RenderingArea() { + const [image, setImage] = useState(""); + + useEffect(() => { + const unsubscribe = window.ipc.on( + buildKey([ID.LIVE_PAINT], { suffix: ":generated" }), + (dataUrl: string) => { + setImage(dataUrl); + } + ); + + return () => { + unsubscribe(); + }; + }, []); + return ( + + + + ); +} + +export function LivePainting() { + const [value, setValue] = useState("side-by-side"); + const isOverlay = value === "overlay"; + return ( + + + { + if (newValue) { + setValue(newValue); + } + }} + > + + + + + + + + + + + + + + ); +} + +export default function Page(_properties: InferGetStaticPropsType) { + const { t } = useTranslation(["common", "labels"]); + return ( + <> + + {`Captain | ${t("labels:livePainting")}`} + + + + + + {t("labels:livePainting")} + + + + + + + + + + + + + ); +} + +export const getStaticProps = makeStaticProperties(["common", "texts", "labels"]); + +export { getStaticPaths } from "@/ions/i18n/get-static"; diff --git a/src/client/public/locales/de/common.json b/src/client/public/locales/de/common.json index 4aff4bf89..772d69a48 100644 --- a/src/client/public/locales/de/common.json +++ b/src/client/public/locales/de/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Suchen & Ersetzen", "seeAll": "Alle ansehen", "selectAll": "Alle auswählen", - "selected": "Ausgewählt", "selectFolder": "Ordner auswählen", "selectImages": "Bilder auswählen", + "selected": "Ausgewählt", "send": "Senden", - "settings": "Einstellungen", "settingUpApp": "Die App wird eingerichtet. Dies kann je nach Server und Internetgeschwindigkeit eine Weile dauern.", + "settings": "Einstellungen", "show": "anzeigen", "submit": "Einreichen", "suffix": "Suffix", diff --git a/src/client/public/locales/de/labels.json b/src/client/public/locales/de/labels.json index 5f5a6d559..9cd71b33b 100644 --- a/src/client/public/locales/de/labels.json +++ b/src/client/public/locales/de/labels.json @@ -4,6 +4,7 @@ "close": "Schließen", "dashboard": "Dashboard", "downloading": "Herunterladen", + "livePainting": "Live-Malerei", "maximize": "Maximieren", "minimize": "Minimieren", "privacy": "Datenschutz", diff --git a/src/client/public/locales/en/common.json b/src/client/public/locales/en/common.json index 230b731a7..2eb06f95a 100644 --- a/src/client/public/locales/en/common.json +++ b/src/client/public/locales/en/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Search & Replace", "seeAll": "See all", "selectAll": "Select All", - "selected": "Selected", "selectFolder": "select folder", "selectImages": "Select Images", + "selected": "Selected", "send": "Send", - "settings": "Settings", "settingUpApp": "Setting up the app. This might take a while depending on server and internet speed.", + "settings": "Settings", "show": "show", "submit": "Submit", "suffix": "Suffix", diff --git a/src/client/public/locales/en/labels.json b/src/client/public/locales/en/labels.json index 0bf72ac36..256a700a7 100644 --- a/src/client/public/locales/en/labels.json +++ b/src/client/public/locales/en/labels.json @@ -4,6 +4,7 @@ "close": "Close", "dashboard": "Dashboard", "downloading": "Downloading", + "livePainting": "Live Painting", "maximize": "Maximize", "minimize": "Minimize", "privacy": "Privacy", diff --git a/src/client/public/locales/es/common.json b/src/client/public/locales/es/common.json index b2b961445..23d7222f1 100644 --- a/src/client/public/locales/es/common.json +++ b/src/client/public/locales/es/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Buscar y reemplazar", "seeAll": "Ver todos", "selectAll": "Seleccionar todo", - "selected": "Seleccionadas", "selectFolder": "seleccionar carpeta", "selectImages": "Seleccionar imágenes", + "selected": "Seleccionadas", "send": "Enviar", - "settings": "Configuración", "settingUpApp": "Configurando la aplicación. Esto puede tardar un tiempo dependiendo del servidor y la velocidad de internet.", + "settings": "Configuración", "show": "mostrar", "submit": "Enviar", "suffix": "Sufijo", diff --git a/src/client/public/locales/es/labels.json b/src/client/public/locales/es/labels.json index 46ae12aeb..9eb82c2e4 100644 --- a/src/client/public/locales/es/labels.json +++ b/src/client/public/locales/es/labels.json @@ -4,6 +4,7 @@ "close": "Cerrar", "dashboard": "Tablero", "downloading": "Descargando", + "livePainting": "Pintura en Vivo", "maximize": "Maximizar", "minimize": "Minimizar", "privacy": "Privacidad", diff --git a/src/client/public/locales/fr/common.json b/src/client/public/locales/fr/common.json index b6029f8f3..7d924cf78 100644 --- a/src/client/public/locales/fr/common.json +++ b/src/client/public/locales/fr/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Rechercher et remplacer", "seeAll": "Voir tout", "selectAll": "Tout sélectionner", - "selected": "Sélectionnés", "selectFolder": "sélectionner un dossier", "selectImages": "Sélectionner des images", + "selected": "Sélectionnés", "send": "Envoyer", - "settings": "Paramètres", "settingUpApp": "Configuration de l'application. Cela peut prendre un certain temps en fonction du serveur et de la vitesse d'internet.", + "settings": "Paramètres", "show": "afficher", "submit": "Soumettre", "suffix": "Suffixe", diff --git a/src/client/public/locales/fr/labels.json b/src/client/public/locales/fr/labels.json index e6517d42f..4b4d0dc65 100644 --- a/src/client/public/locales/fr/labels.json +++ b/src/client/public/locales/fr/labels.json @@ -4,6 +4,7 @@ "close": "Fermer", "dashboard": "Tableau de bord", "downloading": "Téléchargement", + "livePainting": "Peinture en Direct", "maximize": "Maximiser", "minimize": "Minimiser", "privacy": "Confidentialité", diff --git a/src/client/public/locales/he/common.json b/src/client/public/locales/he/common.json index d330cc5bc..7e0432bb7 100644 --- a/src/client/public/locales/he/common.json +++ b/src/client/public/locales/he/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "חפש והחלף", "seeAll": "ראה הכל", "selectAll": "בחר הכל", - "selected": "נבחרים", "selectFolder": "בחר תיקייה", "selectImages": "בחר תמונות", + "selected": "נבחרים", "send": "שלח", - "settings": "הגדרות", "settingUpApp": "מכינים את האפליקציה. זה עשוי לקחת זמן בהתאם למהירות האינטרנט והשרת.", + "settings": "הגדרות", "show": "הצג", "submit": "שלח", "suffix": "סיומת", diff --git a/src/client/public/locales/he/labels.json b/src/client/public/locales/he/labels.json index 4429fbeba..13f3df05b 100644 --- a/src/client/public/locales/he/labels.json +++ b/src/client/public/locales/he/labels.json @@ -4,6 +4,7 @@ "close": "סגור", "dashboard": "לוח בקרה", "downloading": "מוריד", + "livePainting": "ציור חי", "maximize": "הגדל", "minimize": "מזער", "privacy": "פרטיות", diff --git a/src/client/public/locales/it/common.json b/src/client/public/locales/it/common.json index 1d342af95..b00ef422c 100644 --- a/src/client/public/locales/it/common.json +++ b/src/client/public/locales/it/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Cerca e sostituisci", "seeAll": "Vedi tutto", "selectAll": "Seleziona tutto", - "selected": "Selezionate", "selectFolder": "seleziona cartella", "selectImages": "Seleziona immagini", + "selected": "Selezionate", "send": "Inviare", - "settings": "Impostazioni", "settingUpApp": "Configurazione dell'app in corso. Potrebbe richiedere del tempo a seconda del server e della velocità di internet.", + "settings": "Impostazioni", "show": "mostra", "submit": "Invia", "suffix": "Suffisso", diff --git a/src/client/public/locales/it/labels.json b/src/client/public/locales/it/labels.json index 16d61ea65..1fc03f8bb 100644 --- a/src/client/public/locales/it/labels.json +++ b/src/client/public/locales/it/labels.json @@ -4,6 +4,7 @@ "close": "Chiudi", "dashboard": "Cruscotto", "downloading": "Scaricamento", + "livePainting": "Pittura Dal Vivo", "maximize": "Massimizza", "minimize": "Minimizza", "privacy": "Privacy", diff --git a/src/client/public/locales/ja/common.json b/src/client/public/locales/ja/common.json index 325724e91..7c29d2336 100644 --- a/src/client/public/locales/ja/common.json +++ b/src/client/public/locales/ja/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "検索 & 置換", "seeAll": "すべてを見る", "selectAll": "すべて選択", - "selected": "選択済み", "selectFolder": "フォルダを選択", "selectImages": "画像を選択", + "selected": "選択済み", "send": "送信する", - "settings": "設定", "settingUpApp": "アプリを設定しています。サーバーとインターネットの速度によっては、時間がかかる場合があります。", + "settings": "設定", "show": "表示", "submit": "提出", "suffix": "接尾辞", diff --git a/src/client/public/locales/ja/labels.json b/src/client/public/locales/ja/labels.json index 42b38325d..2e057b922 100644 --- a/src/client/public/locales/ja/labels.json +++ b/src/client/public/locales/ja/labels.json @@ -4,6 +4,7 @@ "close": "閉じる", "dashboard": "ダッシュボード", "downloading": "ダウンロード中", + "livePainting": "ライブペインティング", "maximize": "最大化", "minimize": "最小化", "privacy": "プライバシー", diff --git a/src/client/public/locales/nl/common.json b/src/client/public/locales/nl/common.json index 283beab32..148078f67 100644 --- a/src/client/public/locales/nl/common.json +++ b/src/client/public/locales/nl/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Zoeken & vervangen", "seeAll": "Bekijk alles", "selectAll": "Alles selecteren", - "selected": "Geselecteerd", "selectFolder": "map selecteren", "selectImages": "Selecteer afbeeldingen", + "selected": "Geselecteerd", "send": "Verzenden", - "settings": "Instellingen", "settingUpApp": "De app wordt ingesteld. Dit kan afhankelijk van de server en internetsnelheid enige tijd duren.", + "settings": "Instellingen", "show": "tonen", "submit": "Verzenden", "suffix": "Achtervoegsel", diff --git a/src/client/public/locales/nl/labels.json b/src/client/public/locales/nl/labels.json index 89b2d02b5..1d0f78c82 100644 --- a/src/client/public/locales/nl/labels.json +++ b/src/client/public/locales/nl/labels.json @@ -4,6 +4,7 @@ "close": "Sluiten", "dashboard": "Dashboard", "downloading": "Downloaden", + "livePainting": "Live Schilderen", "maximize": "Maximaliseren", "minimize": "Minimaliseren", "privacy": "Privacy", diff --git a/src/client/public/locales/pl/common.json b/src/client/public/locales/pl/common.json index b0e8232df..4502e9dbc 100644 --- a/src/client/public/locales/pl/common.json +++ b/src/client/public/locales/pl/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Szukaj i zamień", "seeAll": "Zobacz wszystkie", "selectAll": "Zaznacz wszystko", - "selected": "Wybrane", "selectFolder": "wybierz folder", "selectImages": "Wybierz obrazy", + "selected": "Wybrane", "send": "Wyślij", - "settings": "Ustawienia", "settingUpApp": "Konfiguracja aplikacji. Może to zająć trochę czasu, w zależności od serwera i prędkości internetu.", + "settings": "Ustawienia", "show": "pokaż", "submit": "Zatwierdź", "suffix": "Sufiks", diff --git a/src/client/public/locales/pl/labels.json b/src/client/public/locales/pl/labels.json index 788722117..46427bf5b 100644 --- a/src/client/public/locales/pl/labels.json +++ b/src/client/public/locales/pl/labels.json @@ -4,6 +4,7 @@ "close": "Zamknij", "dashboard": "Pulpit", "downloading": "Pobieranie", + "livePainting": "Malowanie na Żywo", "maximize": "Zmaksymalizuj", "minimize": "Zminimalizuj", "privacy": "Prywatność", diff --git a/src/client/public/locales/pt/common.json b/src/client/public/locales/pt/common.json index 909561917..11cbfd8c4 100644 --- a/src/client/public/locales/pt/common.json +++ b/src/client/public/locales/pt/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Pesquisar e substituir", "seeAll": "Ver todos", "selectAll": "Selecionar tudo", - "selected": "Selecionadas", "selectFolder": "selecionar pasta", "selectImages": "Selecionar imagens", + "selected": "Selecionadas", "send": "Enviar", - "settings": "Configurações", "settingUpApp": "Configurando o aplicativo. Isso pode levar um tempo dependendo do servidor e da velocidade da internet.", + "settings": "Configurações", "show": "mostrar", "submit": "Submeter", "suffix": "Sufixo", diff --git a/src/client/public/locales/pt/labels.json b/src/client/public/locales/pt/labels.json index 5ea587a04..2b01b86fd 100644 --- a/src/client/public/locales/pt/labels.json +++ b/src/client/public/locales/pt/labels.json @@ -4,6 +4,7 @@ "close": "Fechar", "dashboard": "Painel de Controle", "downloading": "Baixando", + "livePainting": "Pintura ao Vivo", "maximize": "Maximizar", "minimize": "Minimizar", "privacy": "Privacidade", diff --git a/src/client/public/locales/ru/common.json b/src/client/public/locales/ru/common.json index 2f9ad0cee..6474742d1 100644 --- a/src/client/public/locales/ru/common.json +++ b/src/client/public/locales/ru/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "Поиск и замена", "seeAll": "Смотреть все", "selectAll": "Выбрать все", - "selected": "Выбранные", "selectFolder": "выбрать папку", "selectImages": "Выбрать изображения", + "selected": "Выбранные", "send": "Отправить", - "settings": "Настройки", "settingUpApp": "Настройка приложения. Это может занять некоторое время в зависимости от сервера и скорости интернета.", + "settings": "Настройки", "show": "показать", "submit": "Отправить", "suffix": "Суффикс", diff --git a/src/client/public/locales/ru/labels.json b/src/client/public/locales/ru/labels.json index 158e57d74..3a5354bd7 100644 --- a/src/client/public/locales/ru/labels.json +++ b/src/client/public/locales/ru/labels.json @@ -4,6 +4,7 @@ "close": "Закрыть", "dashboard": "Панель управления", "downloading": "Загрузка", + "livePainting": "Живая Живопись", "maximize": "Развернуть", "minimize": "Свернуть", "privacy": "Конфиденциальность", diff --git a/src/client/public/locales/zh/common.json b/src/client/public/locales/zh/common.json index e38663015..ee0741016 100644 --- a/src/client/public/locales/zh/common.json +++ b/src/client/public/locales/zh/common.json @@ -100,12 +100,12 @@ "searchAndReplace": "搜索与替换", "seeAll": "查看全部", "selectAll": "全选", - "selected": "已选", "selectFolder": "选择文件夹", "selectImages": "选择图片", + "selected": "已选", "send": "发送", - "settings": "设置", "settingUpApp": "正在设置应用程序。这可能会根据服务器和互联网速度而需要一段时间。", + "settings": "设置", "show": "显示", "submit": "提交", "suffix": "后缀", diff --git a/src/client/public/locales/zh/labels.json b/src/client/public/locales/zh/labels.json index 25b813d3b..c8d75a16a 100644 --- a/src/client/public/locales/zh/labels.json +++ b/src/client/public/locales/zh/labels.json @@ -4,6 +4,7 @@ "close": "关闭", "dashboard": "仪表板", "downloading": "下载中", + "livePainting": "实时绘画", "maximize": "最大化", "minimize": "最小化", "privacy": "隐私", diff --git a/src/electron/future/ipc/listeners.ts b/src/electron/future/ipc/listeners.ts index 6507d01cd..3775250f8 100644 --- a/src/electron/future/ipc/listeners.ts +++ b/src/electron/future/ipc/listeners.ts @@ -1,5 +1,9 @@ +import fsp from "node:fs/promises"; + import { BrowserWindow, ipcMain } from "electron"; import { download } from "electron-dl"; +import type { ExecaChildProcess } from "execa"; +import { execa } from "execa"; import { buildKey } from "#/build-key"; import { DownloadState, ID } from "#/enums"; @@ -80,3 +84,128 @@ ipcMain.on(buildKey([ID.INSTALL], { suffix: "start" }), async () => { ipcMain.on(buildKey([ID.USER], { suffix: ":language" }), (_event, language) => { userStore.set("language", language); }); + +let process: ExecaChildProcess | undefined; +let cache = ""; + +ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":dataUrl" }), async (_event, dataUrl) => { + const dataString = dataUrl.toString(); + const base64Data = dataString.replace(/^data:image\/png;base64,/, ""); + const decodedImageData = Buffer.from(base64Data, "base64"); + + await fsp.writeFile(getCaptainData("temp/live-painting/input.png"), decodedImageData); +}); + +ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":start" }), () => { + const window_ = BrowserWindow.getFocusedWindow(); + if (!window_) { + return; + } + + if (!process) { + const pythonBinaryPath = getCaptainData("python-embedded/python.exe"); + const scriptPath = getDirectory("python/live-painting/main.py"); + const scriptArguments = [ + "--model_path", + getCaptainData("downloads/stable-diffusion/checkpoint/sd-turbo/"), + "--vae_path", + getCaptainData("downloads/stable-diffusion/vae/taesd/"), + "--input_image_path", + getCaptainData("temp/live-painting/input.png"), + "--output_image_path", + getCaptainData("temp/live-painting/output.png"), + "--disable_stablefast", + // "--debug", + ]; + + process = execa(pythonBinaryPath, ["-u", scriptPath, ...scriptArguments]); + + if (process.stdout && process.stderr) { + process.stdout.on("data", async data => { + const dataString = data.toString(); + + try { + const jsonData = JSON.parse(dataString); + + console.log(`live-painting: ${JSON.stringify(jsonData)}`); + + if (process && jsonData.status === "starting") { + window_.webContents.send( + buildKey([ID.LIVE_PAINT], { suffix: ":starting" }), + true + ); + } + + if (process && jsonData.status === "started") { + window_.webContents.send( + buildKey([ID.LIVE_PAINT], { suffix: ":started" }), + true + ); + } + + if ( + process && + (jsonData.status === "shutdown" || jsonData.status === "stopped") + ) { + if (process) { + if (process.stdout) { + process.stdout.removeAllListeners("data"); + } + + if (process.stderr) { + process.stderr.removeAllListeners("data"); + } + + if (process && !process.killed) { + process.kill(); + } + } + + process = undefined; + + window_.webContents.send( + buildKey([ID.LIVE_PAINT], { suffix: ":stopped" }), + true + ); + } + + if (jsonData.status === "image_generated") { + const imageData = await fsp.readFile( + getCaptainData("temp/live-painting/output.png") + ); + const base64Image = imageData.toString("base64"); + + if (!base64Image.trim()) { + return; + } + + if (base64Image.trim() === cache) { + return; + } + + cache = base64Image; + + window_.webContents.send( + buildKey([ID.LIVE_PAINT], { suffix: ":generated" }), + `data:image/png;base64,${base64Image}` + ); + } + } catch { + console.log("Received non-JSON data:", dataString); + } + }); + + process.stderr.on("data", data => { + console.error(`error: ${data}`); + + window_.webContents.send(buildKey([ID.LIVE_PAINT], { suffix: ":error" }), data); + }); + } + } +}); + +ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":stop" }), () => { + if (process && process.stdin) { + process.stdin.write(JSON.stringify({ command: "shutdown" }) + "\n"); + } +}); diff --git a/src/shared/enums.ts b/src/shared/enums.ts index 248cafbff..223034d49 100644 --- a/src/shared/enums.ts +++ b/src/shared/enums.ts @@ -15,6 +15,7 @@ export enum ID { STORE = "STORE", USER = "USER", WINDOW = "WINDOW", + LIVE_PAINT = "LIVE_PAINT", } /**