diff --git a/playwright/live-painting.test.ts b/playwright/live-painting.test.ts
new file mode 100644
index 000000000..25d11fc74
--- /dev/null
+++ b/playwright/live-painting.test.ts
@@ -0,0 +1,24 @@
+import type { ElectronApplication, Page } from "@playwright/test";
+import { test, expect } from "@playwright/test";
+import { _electron as electron } from "playwright";
+
+let electronApp: ElectronApplication;
+let page: Page;
+
+test.beforeAll(async () => {
+ electronApp = await electron.launch({ args: ["."] });
+ const isPackaged = await electronApp.evaluate(async ({ app }) => app.isPackaged);
+
+ expect(isPackaged).toBe(false);
+});
+
+test.afterAll(async () => {
+ await electronApp.close();
+});
+
+test.skip("Open Live Painting", async () => {
+ page = await electronApp.firstWindow();
+
+ await page.getByTestId("sidebar-live-painting").click();
+ await expect(page.url()).toContain("live-painting");
+});
diff --git a/resources/python/live-painting/main.py b/resources/python/live-painting/main.py
index 990ea3e98..989ca928f 100644
--- a/resources/python/live-painting/main.py
+++ b/resources/python/live-painting/main.py
@@ -1,14 +1,41 @@
-from diffusers import (
- StableDiffusionImg2ImgPipeline,
- AutoencoderTiny,
- EulerAncestralDiscreteScheduler,
-)
+import io
+import sys
+from contextlib import contextmanager
+import json
+
+print(json.dumps({"status": "starting"}))
+
+
+@contextmanager
+def suppress_print(debug=False):
+ # Hide messages printed to stdout / stderr
+
+ if not debug:
+ original_stdout = sys.stdout
+ original_stderr = sys.stderr
+ sys.stdout = io.StringIO()
+ sys.stderr = io.StringIO()
+
+ try:
+ yield
+ finally:
+ sys.stdout = original_stdout
+ sys.stderr = original_stderr
+ else:
+ yield
+
+
+# Get rid of message, that Triton is not available
+with suppress_print():
+ from diffusers import (
+ StableDiffusionImg2ImgPipeline,
+ AutoencoderTiny,
+ EulerAncestralDiscreteScheduler,
+ )
import torch
import time
from PIL import Image
import queue
-import sys
-import json
import threading
import os
import time
@@ -16,15 +43,74 @@
import torch
from diffusers.utils import load_image
from sfast.compilers.diffusion_pipeline_compiler import compile, CompilationConfig
+import argparse
-def read_stdin_loop():
+# Torch optimizations
+torch.set_grad_enabled(False)
+torch.backends.cuda.matmul.allow_tf32 = True
+torch.backends.cudnn.allow_tf32 = True
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Control model and paths via CLI.")
+ parser.add_argument(
+ "--model_path",
+ type=str,
+ help="Path to the model directory or identifier.",
+ required=True,
+ )
+ parser.add_argument(
+ "--vae_path",
+ type=str,
+ help="Path to the vae directory or identifier.",
+ required=True,
+ )
+ parser.add_argument(
+ "--input_image_path",
+ type=str,
+ default="live-canvas-frontend-user-data.png",
+ help="Path to the input image file.",
+ required=True,
+ )
+ parser.add_argument(
+ "--output_image_path",
+ type=str,
+ default="live-canvas-generate-image-output.png",
+ help="Path for the output image file.",
+ )
+ parser.add_argument(
+ "--disable_stablefast",
+ action="store_true",
+ help="Disable the stablefast compilation for faster loading during development.",
+ )
+ parser.add_argument(
+ "--debug",
+ action="store_true",
+ help="Enable debug output.",
+ )
+
+ args = parser.parse_args()
+ return args
+
+
+def read_stdin_loop(params_queue, shutdown_event):
for line in sys.stdin:
if line.strip():
try:
- params_queue.put(json.loads(line))
+ data = json.loads(line)
+
+ if data.get("command") == "shutdown":
+ shutdown_event.set()
+ break
+ else:
+ params_queue.put(data)
except json.JSONDecodeError as e:
- print(f"Error decoding JSON: {e}")
+ print(
+ json.dumps(
+ {"status": "error", "message": f"Error decoding JSON: {e}"}
+ )
+ )
def safe_rename_with_retries(src, dst, max_retries=5, delay=0.005):
@@ -68,134 +154,126 @@ def calculate_min_inference_steps(strength):
return math.ceil(1.0 / strength)
-# Initial/default values for parameters
-prompt = None
-seed = None
-strength = None
-guidance_scale = None
-input_path = "live-canvas-frontend-user-data.png"
-output_path = "live-canvas-generate-image-output.png"
+def prepare_pipeline(model_path, vae_path, disable_stablefast=False):
+ pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
+ model_path,
+ torch_dtype=torch.float16,
+ variant="fp16",
+ safety_checker=None,
+ requires_safety_checker=False,
+ )
-# Queue to hold parameters received from stdin
-params_queue = queue.Queue()
+ pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
+ pipe.safety_checker = None
+ pipe.to(torch.device("cuda"))
-# Start the stdin reading thread
-stdin_thread = threading.Thread(target=read_stdin_loop, daemon=True)
-stdin_thread.start()
+ # Load the stable-fast default config
+ config = CompilationConfig.Default()
+ # Whether to preserve parameters when freezing the model.
+ # If True, parameters will be preserved, but the model will be a bit slower.
+ # If False, parameters will be marked as constants, and the model will be faster.
+ config.preserve_parameters = False
-# Torch optimizations
-torch.set_grad_enabled(False)
-torch.backends.cuda.matmul.allow_tf32 = True
-torch.backends.cudnn.allow_tf32 = True
-
+ # xformers and Triton are suggested for achieving best performance.
+ try:
+ import xformers
-pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
- "stabilityai/sd-turbo",
- torch_dtype=torch.float16,
- variant="fp16",
- safety_checker=None,
- requires_safety_checker=False,
-)
-
-pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
-pipe.safety_checker = None
-pipe.to(torch.device("cuda"))
-
-# Load the stable-fast default config
-config = CompilationConfig.Default()
-
-# Whether to preserve parameters when freezing the model.
-# If True, parameters will be preserved, but the model will be a bit slower.
-# If False, parameters will be marked as constants, and the model will be faster.
-config.preserve_parameters = False
-
-# xformers and Triton are suggested for achieving best performance.
-try:
- import xformers
-
- config.enable_xformers = True
-except ImportError:
- print("xformers not installed, skip")
-try:
- import triton
-
- config.enable_triton = True
-except ImportError:
- print("Triton not installed, skip")
-
-# CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead.
-# But it can increase the amount of GPU memory used.
-config.enable_cuda_graph = True
-
-pipe.width = 512
-pipe.height = 512
-
-pipe.vae = AutoencoderTiny.from_pretrained("madebyollin/taesd").to(
- device=pipe.device, dtype=pipe.dtype
-)
-
-# Channels-last memory format
-# see https://huggingface.co/docs/diffusers/optimization/memory#channelslast-memory-format
-pipe.unet.to(memory_format=torch.channels_last)
-pipe.vae.to(memory_format=torch.channels_last)
-
-# Disable inference progress bar
-pipe.set_progress_bar_config(leave=False)
-pipe.set_progress_bar_config(disable=True)
-
-pipe = compile(pipe, config)
-
-# Warmup
-print("warmup started")
-for _ in range(10):
- init_image = load_image_with_retry(input_path)
- output_image = pipe(
- prompt="the moon, 4k",
- image=init_image,
- height=512,
- width=512,
- num_inference_steps=1,
- num_images_per_prompt=1,
- strength=1.0,
- guidance_scale=0.0,
- ).images[0]
-print("warmup done")
-
-
-def main():
- global prompt, seed, strength, guidance_scale, input_path, output_path
-
- while True:
+ config.enable_xformers = True
+ except ImportError:
+ print("xformers not installed, skip")
+ try:
+ import triton
+
+ config.enable_triton = True
+ except ImportError:
+ print("Triton not installed, skip")
+
+ # CUDA Graph is suggested for small batch sizes and small resolutions to reduce CPU overhead.
+ # But it can increase the amount of GPU memory used.
+ config.enable_cuda_graph = True
+
+ pipe.width = 512
+ pipe.height = 512
+
+ pipe.vae = AutoencoderTiny.from_pretrained(vae_path).to(
+ device=pipe.device, dtype=pipe.dtype
+ )
+
+ # Channels-last memory format
+ # see https://huggingface.co/docs/diffusers/optimization/memory#channelslast-memory-format
+ pipe.unet.to(memory_format=torch.channels_last)
+ pipe.vae.to(memory_format=torch.channels_last)
+
+ # Disable inference progress bar
+ pipe.set_progress_bar_config(leave=False)
+ pipe.set_progress_bar_config(disable=True)
+
+ # Disable stable fast for faster startup of the script, but slower inference
+ if not disable_stablefast:
+ pipe = compile(pipe, config)
+
+ return pipe
+
+
+def warmup(pipe, input_image_path):
+ # Warmup
+ for _ in range(6):
+ init_image = load_image_with_retry(input_image_path)
+ pipe(
+ prompt="the moon, 4k",
+ image=init_image,
+ height=512,
+ width=512,
+ num_inference_steps=1,
+ num_images_per_prompt=1,
+ strength=1.0,
+ guidance_scale=0.0,
+ ).images[0]
+
+
+def main(pipe, input_image_path, output_image_path, shutdown_event):
+ # Initial/default values for parameters
+ prompt = "realistic face, 4k"
+ seed = 1
+ strength = 0.99
+ guidance_scale = None
+
+ # Queue to hold parameters received from stdin
+ params_queue = queue.Queue()
+
+ # Read from stdin
+ stdin_thread = threading.Thread(
+ target=read_stdin_loop, args=(params_queue, shutdown_event), daemon=True
+ )
+ stdin_thread.start()
+
+ while not shutdown_event.is_set():
try:
# Update parameters if new ones are available
while not params_queue.empty():
parameters = params_queue.get_nowait()
prompt = parameters.get("prompt", prompt)
seed = parameters.get("seed", seed)
- input_path = parameters.get("input_path", input_path)
strength = parameters.get("strength", strength)
guidance_scale = parameters.get("guidance_scale", guidance_scale)
- output_path = parameters.get("output_path", output_path)
print(f"Updated parameters {parameters}")
except queue.Empty:
pass # No new parameters, proceed with the existing ones
# Only generate an image if the prompt is not empty
if prompt is not None and prompt.strip():
- start_time = time.time()
-
torch.manual_seed(seed)
- init_image = load_image_with_retry(input_path)
+ init_image = load_image_with_retry(input_image_path)
# Image couldn't be loaded, skip this iteration
if init_image is None:
continue
strength_ = float(strength)
- guidance_scale_ = float(guidance_scale)
- denoise_steps_ = calculate_min_inference_steps(strength_)
+ # guidance_scale_ = float(guidance_scale)
+ # denoise_steps_ = calculate_min_inference_steps(strength_)
image = pipe(
prompt,
@@ -208,35 +286,39 @@ def main():
guidance_scale=1.5,
).images[0]
- end_time = time.time()
-
# Save file
- image.save(f"{output_path}.tmp.png")
- safe_rename_with_retries(f"{output_path}.tmp.png", output_path)
+ image.save(f"{output_image_path}.tmp.png")
+ safe_rename_with_retries(f"{output_image_path}.tmp.png", output_image_path)
+
+ print(json.dumps({"status": "image_generated"}))
- duration = (end_time - start_time) * 1000
- # print(f"{duration:.2f} ms")
- times.append(duration)
else:
image = Image.new("RGB", (512, 512), color="white")
- image.save(f"{output_path}.tmp.png")
- safe_rename_with_retries(f"{output_path}.tmp.png", output_path)
+ image.save(f"{output_image_path}.tmp.png")
+ safe_rename_with_retries(f"{output_image_path}.tmp.png", output_image_path)
if __name__ == "__main__":
- times = []
+ # Used for gracefully shutting down the script
+ shutdown_event = threading.Event()
try:
- print("started loop")
- main()
+ args = parse_args()
+ pipe = None
+
+ with suppress_print(args.debug):
+ pipe = prepare_pipeline(
+ args.model_path, args.vae_path, args.disable_stablefast
+ )
+ warmup(pipe, args.input_image_path)
+
+ print(json.dumps({"status": "started"}))
+
+ main(pipe, args.input_image_path, args.output_image_path, shutdown_event)
+
+ print(json.dumps({"status": "shutdown"}))
+
except Exception as error:
- print(error)
+ print(json.dumps({"status": "error", "message": str(error)}))
except:
- if times:
- total_time = sum(times)
- average_time = total_time / len(times)
- print(
- f"Total time: {total_time:.2f} ms, Total executions: {len(times)}, Average time: {average_time:.2f} ms"
- )
- else:
- print(f"No operations were performed or exception")
+ print(json.dumps({"status": "stopped"}))
diff --git a/resources/python/live-painting/test-resources/input.png b/resources/python/live-painting/test-resources/input.png
new file mode 100644
index 000000000..5f52c3a2e
Binary files /dev/null and b/resources/python/live-painting/test-resources/input.png differ
diff --git a/src/client/organisms/layout/index.tsx b/src/client/organisms/layout/index.tsx
index 151a16dcc..ac2920ba9 100644
--- a/src/client/organisms/layout/index.tsx
+++ b/src/client/organisms/layout/index.tsx
@@ -50,8 +50,12 @@ export function Layout({ children }: { children?: ReactNode }) {
}>
{t("common:training")}
- }>
- {t("common:livePainting")}
+ }
+ data-testid="sidebar-live-painting"
+ >
+ {t("labels:livePainting")}
(null);
+ const context = useRef(null);
+ const isDrawing = useRef(false);
+ const [, setImage] = useAtom(imageAtom);
+
+ function startDrawing(event: ReactPointerEvent) {
+ if (!canvas.current) {
+ return;
+ }
+
+ isDrawing.current = true;
+ const rect = canvas.current.getBoundingClientRect();
+ context.current?.beginPath();
+ context.current?.moveTo(event.clientX - rect.left, event.clientY - rect.top);
+ }
+
+ useEffect(() => {
+ const canvasElement = canvas.current;
+ let animationFame: number;
+ if (!canvasElement) {
+ return;
+ }
+
+ canvasElement.height = 512;
+ canvasElement.width = 512;
+ context.current = canvasElement.getContext("2d");
+
+ function draw(event: MouseEvent) {
+ if (!isDrawing.current || !canvas.current) {
+ return;
+ }
+
+ const rect = canvas.current.getBoundingClientRect();
+ context.current?.lineTo(event.clientX - rect.left, event.clientY - rect.top);
+ context.current?.stroke();
+ const dataUrl = canvas.current.toDataURL();
+ setImage(dataUrl);
+ window.ipc.send(buildKey([ID.LIVE_PAINT], { suffix: ":dataUrl" }), dataUrl);
+ }
+
+ function handleMouseUp() {
+ if (isDrawing.current) {
+ context.current?.closePath();
+ isDrawing.current = false;
+ }
+
+ cancelAnimationFrame(animationFame);
+ }
+
+ function handleMouseMove(event: MouseEvent) {
+ animationFame = requestAnimationFrame(() => {
+ draw(event);
+ });
+ }
+
+ if (context.current) {
+ context.current.fillStyle = "#ffffff";
+ context.current.rect(0, 0, canvasElement.width, canvasElement.height);
+ context.current.fill();
+ context.current.strokeStyle = "#000000";
+ context.current.lineWidth = 5;
+ context.current.lineJoin = "round";
+ context.current.lineCap = "round";
+
+ document.addEventListener("mousemove", handleMouseMove, { passive: true });
+ document.addEventListener("mouseup", handleMouseUp);
+ }
+
+ return () => {
+ document.removeEventListener("mousemove", handleMouseMove);
+ document.removeEventListener("mouseup", handleMouseUp);
+ cancelAnimationFrame(animationFame);
+ };
+ }, [setImage]);
+
+ return (
+
+
+
+ );
+}
+
+function RenderingArea() {
+ const [image, setImage] = useState("");
+
+ useEffect(() => {
+ const unsubscribe = window.ipc.on(
+ buildKey([ID.LIVE_PAINT], { suffix: ":generated" }),
+ (dataUrl: string) => {
+ setImage(dataUrl);
+ }
+ );
+
+ return () => {
+ unsubscribe();
+ };
+ }, []);
+ return (
+
+
+
+ );
+}
+
+export function LivePainting() {
+ const [value, setValue] = useState("side-by-side");
+ const isOverlay = value === "overlay";
+ return (
+
+
+ {
+ if (newValue) {
+ setValue(newValue);
+ }
+ }}
+ >
+
+
+
+
+
+
+
+
+
+
+
+
+
+ );
+}
+
+export default function Page(_properties: InferGetStaticPropsType) {
+ const { t } = useTranslation(["common", "labels"]);
+ return (
+ <>
+
+ {`Captain | ${t("labels:livePainting")}`}
+
+
+
+
+
+ {t("labels:livePainting")}
+
+
+
+
+
+
+
+
+
+
+
+ >
+ );
+}
+
+export const getStaticProps = makeStaticProperties(["common", "texts", "labels"]);
+
+export { getStaticPaths } from "@/ions/i18n/get-static";
diff --git a/src/client/public/locales/de/common.json b/src/client/public/locales/de/common.json
index 4aff4bf89..772d69a48 100644
--- a/src/client/public/locales/de/common.json
+++ b/src/client/public/locales/de/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Suchen & Ersetzen",
"seeAll": "Alle ansehen",
"selectAll": "Alle auswählen",
- "selected": "Ausgewählt",
"selectFolder": "Ordner auswählen",
"selectImages": "Bilder auswählen",
+ "selected": "Ausgewählt",
"send": "Senden",
- "settings": "Einstellungen",
"settingUpApp": "Die App wird eingerichtet. Dies kann je nach Server und Internetgeschwindigkeit eine Weile dauern.",
+ "settings": "Einstellungen",
"show": "anzeigen",
"submit": "Einreichen",
"suffix": "Suffix",
diff --git a/src/client/public/locales/de/labels.json b/src/client/public/locales/de/labels.json
index 5f5a6d559..9cd71b33b 100644
--- a/src/client/public/locales/de/labels.json
+++ b/src/client/public/locales/de/labels.json
@@ -4,6 +4,7 @@
"close": "Schließen",
"dashboard": "Dashboard",
"downloading": "Herunterladen",
+ "livePainting": "Live-Malerei",
"maximize": "Maximieren",
"minimize": "Minimieren",
"privacy": "Datenschutz",
diff --git a/src/client/public/locales/en/common.json b/src/client/public/locales/en/common.json
index 230b731a7..2eb06f95a 100644
--- a/src/client/public/locales/en/common.json
+++ b/src/client/public/locales/en/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Search & Replace",
"seeAll": "See all",
"selectAll": "Select All",
- "selected": "Selected",
"selectFolder": "select folder",
"selectImages": "Select Images",
+ "selected": "Selected",
"send": "Send",
- "settings": "Settings",
"settingUpApp": "Setting up the app. This might take a while depending on server and internet speed.",
+ "settings": "Settings",
"show": "show",
"submit": "Submit",
"suffix": "Suffix",
diff --git a/src/client/public/locales/en/labels.json b/src/client/public/locales/en/labels.json
index 0bf72ac36..256a700a7 100644
--- a/src/client/public/locales/en/labels.json
+++ b/src/client/public/locales/en/labels.json
@@ -4,6 +4,7 @@
"close": "Close",
"dashboard": "Dashboard",
"downloading": "Downloading",
+ "livePainting": "Live Painting",
"maximize": "Maximize",
"minimize": "Minimize",
"privacy": "Privacy",
diff --git a/src/client/public/locales/es/common.json b/src/client/public/locales/es/common.json
index b2b961445..23d7222f1 100644
--- a/src/client/public/locales/es/common.json
+++ b/src/client/public/locales/es/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Buscar y reemplazar",
"seeAll": "Ver todos",
"selectAll": "Seleccionar todo",
- "selected": "Seleccionadas",
"selectFolder": "seleccionar carpeta",
"selectImages": "Seleccionar imágenes",
+ "selected": "Seleccionadas",
"send": "Enviar",
- "settings": "Configuración",
"settingUpApp": "Configurando la aplicación. Esto puede tardar un tiempo dependiendo del servidor y la velocidad de internet.",
+ "settings": "Configuración",
"show": "mostrar",
"submit": "Enviar",
"suffix": "Sufijo",
diff --git a/src/client/public/locales/es/labels.json b/src/client/public/locales/es/labels.json
index 46ae12aeb..9eb82c2e4 100644
--- a/src/client/public/locales/es/labels.json
+++ b/src/client/public/locales/es/labels.json
@@ -4,6 +4,7 @@
"close": "Cerrar",
"dashboard": "Tablero",
"downloading": "Descargando",
+ "livePainting": "Pintura en Vivo",
"maximize": "Maximizar",
"minimize": "Minimizar",
"privacy": "Privacidad",
diff --git a/src/client/public/locales/fr/common.json b/src/client/public/locales/fr/common.json
index b6029f8f3..7d924cf78 100644
--- a/src/client/public/locales/fr/common.json
+++ b/src/client/public/locales/fr/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Rechercher et remplacer",
"seeAll": "Voir tout",
"selectAll": "Tout sélectionner",
- "selected": "Sélectionnés",
"selectFolder": "sélectionner un dossier",
"selectImages": "Sélectionner des images",
+ "selected": "Sélectionnés",
"send": "Envoyer",
- "settings": "Paramètres",
"settingUpApp": "Configuration de l'application. Cela peut prendre un certain temps en fonction du serveur et de la vitesse d'internet.",
+ "settings": "Paramètres",
"show": "afficher",
"submit": "Soumettre",
"suffix": "Suffixe",
diff --git a/src/client/public/locales/fr/labels.json b/src/client/public/locales/fr/labels.json
index e6517d42f..4b4d0dc65 100644
--- a/src/client/public/locales/fr/labels.json
+++ b/src/client/public/locales/fr/labels.json
@@ -4,6 +4,7 @@
"close": "Fermer",
"dashboard": "Tableau de bord",
"downloading": "Téléchargement",
+ "livePainting": "Peinture en Direct",
"maximize": "Maximiser",
"minimize": "Minimiser",
"privacy": "Confidentialité",
diff --git a/src/client/public/locales/he/common.json b/src/client/public/locales/he/common.json
index d330cc5bc..7e0432bb7 100644
--- a/src/client/public/locales/he/common.json
+++ b/src/client/public/locales/he/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "חפש והחלף",
"seeAll": "ראה הכל",
"selectAll": "בחר הכל",
- "selected": "נבחרים",
"selectFolder": "בחר תיקייה",
"selectImages": "בחר תמונות",
+ "selected": "נבחרים",
"send": "שלח",
- "settings": "הגדרות",
"settingUpApp": "מכינים את האפליקציה. זה עשוי לקחת זמן בהתאם למהירות האינטרנט והשרת.",
+ "settings": "הגדרות",
"show": "הצג",
"submit": "שלח",
"suffix": "סיומת",
diff --git a/src/client/public/locales/he/labels.json b/src/client/public/locales/he/labels.json
index 4429fbeba..13f3df05b 100644
--- a/src/client/public/locales/he/labels.json
+++ b/src/client/public/locales/he/labels.json
@@ -4,6 +4,7 @@
"close": "סגור",
"dashboard": "לוח בקרה",
"downloading": "מוריד",
+ "livePainting": "ציור חי",
"maximize": "הגדל",
"minimize": "מזער",
"privacy": "פרטיות",
diff --git a/src/client/public/locales/it/common.json b/src/client/public/locales/it/common.json
index 1d342af95..b00ef422c 100644
--- a/src/client/public/locales/it/common.json
+++ b/src/client/public/locales/it/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Cerca e sostituisci",
"seeAll": "Vedi tutto",
"selectAll": "Seleziona tutto",
- "selected": "Selezionate",
"selectFolder": "seleziona cartella",
"selectImages": "Seleziona immagini",
+ "selected": "Selezionate",
"send": "Inviare",
- "settings": "Impostazioni",
"settingUpApp": "Configurazione dell'app in corso. Potrebbe richiedere del tempo a seconda del server e della velocità di internet.",
+ "settings": "Impostazioni",
"show": "mostra",
"submit": "Invia",
"suffix": "Suffisso",
diff --git a/src/client/public/locales/it/labels.json b/src/client/public/locales/it/labels.json
index 16d61ea65..1fc03f8bb 100644
--- a/src/client/public/locales/it/labels.json
+++ b/src/client/public/locales/it/labels.json
@@ -4,6 +4,7 @@
"close": "Chiudi",
"dashboard": "Cruscotto",
"downloading": "Scaricamento",
+ "livePainting": "Pittura Dal Vivo",
"maximize": "Massimizza",
"minimize": "Minimizza",
"privacy": "Privacy",
diff --git a/src/client/public/locales/ja/common.json b/src/client/public/locales/ja/common.json
index 325724e91..7c29d2336 100644
--- a/src/client/public/locales/ja/common.json
+++ b/src/client/public/locales/ja/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "検索 & 置換",
"seeAll": "すべてを見る",
"selectAll": "すべて選択",
- "selected": "選択済み",
"selectFolder": "フォルダを選択",
"selectImages": "画像を選択",
+ "selected": "選択済み",
"send": "送信する",
- "settings": "設定",
"settingUpApp": "アプリを設定しています。サーバーとインターネットの速度によっては、時間がかかる場合があります。",
+ "settings": "設定",
"show": "表示",
"submit": "提出",
"suffix": "接尾辞",
diff --git a/src/client/public/locales/ja/labels.json b/src/client/public/locales/ja/labels.json
index 42b38325d..2e057b922 100644
--- a/src/client/public/locales/ja/labels.json
+++ b/src/client/public/locales/ja/labels.json
@@ -4,6 +4,7 @@
"close": "閉じる",
"dashboard": "ダッシュボード",
"downloading": "ダウンロード中",
+ "livePainting": "ライブペインティング",
"maximize": "最大化",
"minimize": "最小化",
"privacy": "プライバシー",
diff --git a/src/client/public/locales/nl/common.json b/src/client/public/locales/nl/common.json
index 283beab32..148078f67 100644
--- a/src/client/public/locales/nl/common.json
+++ b/src/client/public/locales/nl/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Zoeken & vervangen",
"seeAll": "Bekijk alles",
"selectAll": "Alles selecteren",
- "selected": "Geselecteerd",
"selectFolder": "map selecteren",
"selectImages": "Selecteer afbeeldingen",
+ "selected": "Geselecteerd",
"send": "Verzenden",
- "settings": "Instellingen",
"settingUpApp": "De app wordt ingesteld. Dit kan afhankelijk van de server en internetsnelheid enige tijd duren.",
+ "settings": "Instellingen",
"show": "tonen",
"submit": "Verzenden",
"suffix": "Achtervoegsel",
diff --git a/src/client/public/locales/nl/labels.json b/src/client/public/locales/nl/labels.json
index 89b2d02b5..1d0f78c82 100644
--- a/src/client/public/locales/nl/labels.json
+++ b/src/client/public/locales/nl/labels.json
@@ -4,6 +4,7 @@
"close": "Sluiten",
"dashboard": "Dashboard",
"downloading": "Downloaden",
+ "livePainting": "Live Schilderen",
"maximize": "Maximaliseren",
"minimize": "Minimaliseren",
"privacy": "Privacy",
diff --git a/src/client/public/locales/pl/common.json b/src/client/public/locales/pl/common.json
index b0e8232df..4502e9dbc 100644
--- a/src/client/public/locales/pl/common.json
+++ b/src/client/public/locales/pl/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Szukaj i zamień",
"seeAll": "Zobacz wszystkie",
"selectAll": "Zaznacz wszystko",
- "selected": "Wybrane",
"selectFolder": "wybierz folder",
"selectImages": "Wybierz obrazy",
+ "selected": "Wybrane",
"send": "Wyślij",
- "settings": "Ustawienia",
"settingUpApp": "Konfiguracja aplikacji. Może to zająć trochę czasu, w zależności od serwera i prędkości internetu.",
+ "settings": "Ustawienia",
"show": "pokaż",
"submit": "Zatwierdź",
"suffix": "Sufiks",
diff --git a/src/client/public/locales/pl/labels.json b/src/client/public/locales/pl/labels.json
index 788722117..46427bf5b 100644
--- a/src/client/public/locales/pl/labels.json
+++ b/src/client/public/locales/pl/labels.json
@@ -4,6 +4,7 @@
"close": "Zamknij",
"dashboard": "Pulpit",
"downloading": "Pobieranie",
+ "livePainting": "Malowanie na Żywo",
"maximize": "Zmaksymalizuj",
"minimize": "Zminimalizuj",
"privacy": "Prywatność",
diff --git a/src/client/public/locales/pt/common.json b/src/client/public/locales/pt/common.json
index 909561917..11cbfd8c4 100644
--- a/src/client/public/locales/pt/common.json
+++ b/src/client/public/locales/pt/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Pesquisar e substituir",
"seeAll": "Ver todos",
"selectAll": "Selecionar tudo",
- "selected": "Selecionadas",
"selectFolder": "selecionar pasta",
"selectImages": "Selecionar imagens",
+ "selected": "Selecionadas",
"send": "Enviar",
- "settings": "Configurações",
"settingUpApp": "Configurando o aplicativo. Isso pode levar um tempo dependendo do servidor e da velocidade da internet.",
+ "settings": "Configurações",
"show": "mostrar",
"submit": "Submeter",
"suffix": "Sufixo",
diff --git a/src/client/public/locales/pt/labels.json b/src/client/public/locales/pt/labels.json
index 5ea587a04..2b01b86fd 100644
--- a/src/client/public/locales/pt/labels.json
+++ b/src/client/public/locales/pt/labels.json
@@ -4,6 +4,7 @@
"close": "Fechar",
"dashboard": "Painel de Controle",
"downloading": "Baixando",
+ "livePainting": "Pintura ao Vivo",
"maximize": "Maximizar",
"minimize": "Minimizar",
"privacy": "Privacidade",
diff --git a/src/client/public/locales/ru/common.json b/src/client/public/locales/ru/common.json
index 2f9ad0cee..6474742d1 100644
--- a/src/client/public/locales/ru/common.json
+++ b/src/client/public/locales/ru/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "Поиск и замена",
"seeAll": "Смотреть все",
"selectAll": "Выбрать все",
- "selected": "Выбранные",
"selectFolder": "выбрать папку",
"selectImages": "Выбрать изображения",
+ "selected": "Выбранные",
"send": "Отправить",
- "settings": "Настройки",
"settingUpApp": "Настройка приложения. Это может занять некоторое время в зависимости от сервера и скорости интернета.",
+ "settings": "Настройки",
"show": "показать",
"submit": "Отправить",
"suffix": "Суффикс",
diff --git a/src/client/public/locales/ru/labels.json b/src/client/public/locales/ru/labels.json
index 158e57d74..3a5354bd7 100644
--- a/src/client/public/locales/ru/labels.json
+++ b/src/client/public/locales/ru/labels.json
@@ -4,6 +4,7 @@
"close": "Закрыть",
"dashboard": "Панель управления",
"downloading": "Загрузка",
+ "livePainting": "Живая Живопись",
"maximize": "Развернуть",
"minimize": "Свернуть",
"privacy": "Конфиденциальность",
diff --git a/src/client/public/locales/zh/common.json b/src/client/public/locales/zh/common.json
index e38663015..ee0741016 100644
--- a/src/client/public/locales/zh/common.json
+++ b/src/client/public/locales/zh/common.json
@@ -100,12 +100,12 @@
"searchAndReplace": "搜索与替换",
"seeAll": "查看全部",
"selectAll": "全选",
- "selected": "已选",
"selectFolder": "选择文件夹",
"selectImages": "选择图片",
+ "selected": "已选",
"send": "发送",
- "settings": "设置",
"settingUpApp": "正在设置应用程序。这可能会根据服务器和互联网速度而需要一段时间。",
+ "settings": "设置",
"show": "显示",
"submit": "提交",
"suffix": "后缀",
diff --git a/src/client/public/locales/zh/labels.json b/src/client/public/locales/zh/labels.json
index 25b813d3b..c8d75a16a 100644
--- a/src/client/public/locales/zh/labels.json
+++ b/src/client/public/locales/zh/labels.json
@@ -4,6 +4,7 @@
"close": "关闭",
"dashboard": "仪表板",
"downloading": "下载中",
+ "livePainting": "实时绘画",
"maximize": "最大化",
"minimize": "最小化",
"privacy": "隐私",
diff --git a/src/electron/future/ipc/listeners.ts b/src/electron/future/ipc/listeners.ts
index 6507d01cd..3775250f8 100644
--- a/src/electron/future/ipc/listeners.ts
+++ b/src/electron/future/ipc/listeners.ts
@@ -1,5 +1,9 @@
+import fsp from "node:fs/promises";
+
import { BrowserWindow, ipcMain } from "electron";
import { download } from "electron-dl";
+import type { ExecaChildProcess } from "execa";
+import { execa } from "execa";
import { buildKey } from "#/build-key";
import { DownloadState, ID } from "#/enums";
@@ -80,3 +84,128 @@ ipcMain.on(buildKey([ID.INSTALL], { suffix: "start" }), async () => {
ipcMain.on(buildKey([ID.USER], { suffix: ":language" }), (_event, language) => {
userStore.set("language", language);
});
+
+let process: ExecaChildProcess | undefined;
+let cache = "";
+
+ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":dataUrl" }), async (_event, dataUrl) => {
+ const dataString = dataUrl.toString();
+ const base64Data = dataString.replace(/^data:image\/png;base64,/, "");
+ const decodedImageData = Buffer.from(base64Data, "base64");
+
+ await fsp.writeFile(getCaptainData("temp/live-painting/input.png"), decodedImageData);
+});
+
+ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":start" }), () => {
+ const window_ = BrowserWindow.getFocusedWindow();
+ if (!window_) {
+ return;
+ }
+
+ if (!process) {
+ const pythonBinaryPath = getCaptainData("python-embedded/python.exe");
+ const scriptPath = getDirectory("python/live-painting/main.py");
+ const scriptArguments = [
+ "--model_path",
+ getCaptainData("downloads/stable-diffusion/checkpoint/sd-turbo/"),
+ "--vae_path",
+ getCaptainData("downloads/stable-diffusion/vae/taesd/"),
+ "--input_image_path",
+ getCaptainData("temp/live-painting/input.png"),
+ "--output_image_path",
+ getCaptainData("temp/live-painting/output.png"),
+ "--disable_stablefast",
+ // "--debug",
+ ];
+
+ process = execa(pythonBinaryPath, ["-u", scriptPath, ...scriptArguments]);
+
+ if (process.stdout && process.stderr) {
+ process.stdout.on("data", async data => {
+ const dataString = data.toString();
+
+ try {
+ const jsonData = JSON.parse(dataString);
+
+ console.log(`live-painting: ${JSON.stringify(jsonData)}`);
+
+ if (process && jsonData.status === "starting") {
+ window_.webContents.send(
+ buildKey([ID.LIVE_PAINT], { suffix: ":starting" }),
+ true
+ );
+ }
+
+ if (process && jsonData.status === "started") {
+ window_.webContents.send(
+ buildKey([ID.LIVE_PAINT], { suffix: ":started" }),
+ true
+ );
+ }
+
+ if (
+ process &&
+ (jsonData.status === "shutdown" || jsonData.status === "stopped")
+ ) {
+ if (process) {
+ if (process.stdout) {
+ process.stdout.removeAllListeners("data");
+ }
+
+ if (process.stderr) {
+ process.stderr.removeAllListeners("data");
+ }
+
+ if (process && !process.killed) {
+ process.kill();
+ }
+ }
+
+ process = undefined;
+
+ window_.webContents.send(
+ buildKey([ID.LIVE_PAINT], { suffix: ":stopped" }),
+ true
+ );
+ }
+
+ if (jsonData.status === "image_generated") {
+ const imageData = await fsp.readFile(
+ getCaptainData("temp/live-painting/output.png")
+ );
+ const base64Image = imageData.toString("base64");
+
+ if (!base64Image.trim()) {
+ return;
+ }
+
+ if (base64Image.trim() === cache) {
+ return;
+ }
+
+ cache = base64Image;
+
+ window_.webContents.send(
+ buildKey([ID.LIVE_PAINT], { suffix: ":generated" }),
+ `data:image/png;base64,${base64Image}`
+ );
+ }
+ } catch {
+ console.log("Received non-JSON data:", dataString);
+ }
+ });
+
+ process.stderr.on("data", data => {
+ console.error(`error: ${data}`);
+
+ window_.webContents.send(buildKey([ID.LIVE_PAINT], { suffix: ":error" }), data);
+ });
+ }
+ }
+});
+
+ipcMain.on(buildKey([ID.LIVE_PAINT], { suffix: ":stop" }), () => {
+ if (process && process.stdin) {
+ process.stdin.write(JSON.stringify({ command: "shutdown" }) + "\n");
+ }
+});
diff --git a/src/shared/enums.ts b/src/shared/enums.ts
index 248cafbff..223034d49 100644
--- a/src/shared/enums.ts
+++ b/src/shared/enums.ts
@@ -15,6 +15,7 @@ export enum ID {
STORE = "STORE",
USER = "USER",
WINDOW = "WINDOW",
+ LIVE_PAINT = "LIVE_PAINT",
}
/**