From 0fa3b4e83269fb19618209f26f5f47261f01b08f Mon Sep 17 00:00:00 2001 From: CI/CD Bot Date: Thu, 7 Mar 2024 17:08:01 +0000 Subject: [PATCH] Update nodes doc --- .../ModelMergeBlockNumber.md | 77 ++++++++++ docs/md/ComfyUI_experiments/ModelMergeSDXL.md | 77 ++++++++++ .../ModelMergeSDXLDetailedTransformers.md | 96 ++++++++++++ .../ModelMergeSDXLTransformers.md | 112 ++++++++++++++ .../ModelSamplerTonemapNoiseTest.md | 66 ++++++++ .../ReferenceOnlySimple.md | 86 +++++++++++ .../RescaleClassifierFreeGuidanceTest.md | 60 ++++++++ .../TonemapNoiseWithRescaleCFG.md | 81 ++++++++++ docs/md/SavePNG/PNG_Save.md | 103 +++++++++++++ docs/md/llmnodes/DocumentNode.md | 144 ++++++++++++++++++ docs/md/llmnodes/DocumentPack.md | 83 ++++++++++ docs/md/llmnodes/LoadOpenAIModel.md | 57 +++++++ mkdocs.yml | 44 +++--- 13 files changed, 1062 insertions(+), 24 deletions(-) create mode 100644 docs/md/ComfyUI_experiments/ModelMergeBlockNumber.md create mode 100644 docs/md/ComfyUI_experiments/ModelMergeSDXL.md create mode 100644 docs/md/ComfyUI_experiments/ModelMergeSDXLDetailedTransformers.md create mode 100644 docs/md/ComfyUI_experiments/ModelMergeSDXLTransformers.md create mode 100644 docs/md/ComfyUI_experiments/ModelSamplerTonemapNoiseTest.md create mode 100644 docs/md/ComfyUI_experiments/ReferenceOnlySimple.md create mode 100644 docs/md/ComfyUI_experiments/RescaleClassifierFreeGuidanceTest.md create mode 100644 docs/md/ComfyUI_experiments/TonemapNoiseWithRescaleCFG.md create mode 100644 docs/md/SavePNG/PNG_Save.md create mode 100644 docs/md/llmnodes/DocumentNode.md create mode 100644 docs/md/llmnodes/DocumentPack.md create mode 100644 docs/md/llmnodes/LoadOpenAIModel.md diff --git a/docs/md/ComfyUI_experiments/ModelMergeBlockNumber.md b/docs/md/ComfyUI_experiments/ModelMergeBlockNumber.md new file mode 100644 index 0000000000..3ee3d11f04 --- /dev/null +++ b/docs/md/ComfyUI_experiments/ModelMergeBlockNumber.md @@ -0,0 +1,77 @@ +# ModelMergeBlockNumber +## Documentation +- Class name: `ModelMergeBlockNumber` +- Category: `advanced/model_merging` +- Output node: `False` + +This node specializes in merging two models by applying a weighted combination of their components. It allows for fine-grained control over the merging process by specifying weights for different parts of the models, such as time embeddings, label embeddings, input blocks, middle blocks, output blocks, and the output layer. This enables the creation of hybrid models that blend features from both input models according to specified ratios. +## Input types +- **`model1`** + - The first model to be merged. It serves as the base model for the merging process. + - Python dtype: `comfy.model_management.Model` + - Comfy dtype: `MODEL` +- **`model2`** + - The second model to be merged. Its components are selectively blended into the first model based on the specified weights. + - Python dtype: `comfy.model_management.Model` + - Comfy dtype: `MODEL` +- **`time_embed.`** + - Specifies the weight for merging the time embedding components of the two models. Allows for precise control over how time-related features are combined. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`label_emb.`** + - Determines the weight for merging the label embedding components. This parameter enables customization of how label-related features are integrated. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i`** + - Controls the weight for merging the i-th input block, where i ranges from 0 to 11. This allows for detailed customization of the initial stages of the model. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`middle_block.i`** + - Sets the weight for merging the i-th middle block, where i ranges from 0 to 2. This parameter influences the blending of the models' intermediate processing stages. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i`** + - Specifies the weight for merging the i-th output block, where i ranges from 0 to 11. It affects how the final stages of the models are combined. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`out.`** + - Determines the weight for merging the output layer components. This parameter finalizes the blending of the models' output features. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The resulting model after merging the specified components of the two input models. + - Python dtype: `comfy.model_management.Model` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +The ModelMergeBlockNumber node is primarily utilized for creating hybrid AI models by merging two distinct models, allowing for the customization of the merging process through weighted parameters for different model components, including time and label embeddings, input, middle, and output blocks, and the output layer. This node takes two models as input and outputs a single model that combines features from both according to specified weights, making it ideal for experiments in model blending and performance enhancement. +## Source code +```python +class ModelMergeBlockNumber(comfy_extras.nodes_model_merging.ModelMergeBlocks): + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + for i in range(12): + arg_dict["input_blocks.{}.".format(i)] = argument + + for i in range(3): + arg_dict["middle_block.{}.".format(i)] = argument + + for i in range(12): + arg_dict["output_blocks.{}.".format(i)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + +``` diff --git a/docs/md/ComfyUI_experiments/ModelMergeSDXL.md b/docs/md/ComfyUI_experiments/ModelMergeSDXL.md new file mode 100644 index 0000000000..ff8b30fcad --- /dev/null +++ b/docs/md/ComfyUI_experiments/ModelMergeSDXL.md @@ -0,0 +1,77 @@ +# ModelMergeSDXL +## Documentation +- Class name: `ModelMergeSDXL` +- Category: `advanced/model_merging` +- Output node: `False` + +This node merges two models by blending their components based on specified ratios for different parts of the models. It allows for fine-tuned control over how much of each model's characteristics are retained in the merged model, by adjusting ratios for time embeddings, label embeddings, input blocks, middle blocks, output blocks, and the output layer. +## Input types +- **`model1`** + - The first model to be merged. It serves as the base model for the merging process. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`model2`** + - The second model to be merged. Its components are blended with the first model based on the specified ratios. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`time_embed.`** + - Specifies the blending ratio for the time embedding components of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`label_emb.`** + - Specifies the blending ratio for the label embedding components of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i`** + - Specifies the blending ratio for the i-th input block of the models, where i ranges from 0 to 8. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`middle_block.i`** + - Specifies the blending ratio for the i-th middle block of the models, where i ranges from 0 to 2. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i`** + - Specifies the blending ratio for the i-th output block of the models, where i ranges from 0 to 8. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`out.`** + - Specifies the blending ratio for the output layer of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The resulting model after merging the two input models based on the specified blending ratios. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `GPU` +- Common nodes: unknown + +The ModelMergeSDXL node is primarily utilized for blending two AI models by adjusting the ratios for specific components like time embeddings, label embeddings, input, middle, and output blocks, and the output layer, making it ideal for creating hybrid models that combine the strengths of both input models. It requires two models as input and outputs a single model that embodies the merged characteristics based on the specified blending ratios. +## Source code +```python +class ModelMergeSDXL(comfy_extras.nodes_model_merging.ModelMergeBlocks): + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + for i in range(9): + arg_dict["input_blocks.{}".format(i)] = argument + + for i in range(3): + arg_dict["middle_block.{}".format(i)] = argument + + for i in range(9): + arg_dict["output_blocks.{}".format(i)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + +``` diff --git a/docs/md/ComfyUI_experiments/ModelMergeSDXLDetailedTransformers.md b/docs/md/ComfyUI_experiments/ModelMergeSDXLDetailedTransformers.md new file mode 100644 index 0000000000..46464ef455 --- /dev/null +++ b/docs/md/ComfyUI_experiments/ModelMergeSDXLDetailedTransformers.md @@ -0,0 +1,96 @@ +# ModelMergeSDXLDetailedTransformers +## Documentation +- Class name: `ModelMergeSDXLDetailedTransformers` +- Category: `advanced/model_merging` +- Output node: `False` + +This node specializes in merging models with a detailed focus on transformer blocks. It inherits from a base model merging class and is designed to handle complex merging scenarios involving transformer components within models. The node likely incorporates advanced techniques to merge specific parts of models, such as transformer blocks, with precision, allowing for fine-tuned control over the merging process. +## Input types +- **`model1`** + - The first model to be merged. It is a crucial component of the merging process, serving as one of the primary sources of features or parameters to be combined. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`model2`** + - The second model to be merged. It plays a significant role in the merging process, contributing its features or parameters to be combined with those of the first model. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`time_embed.`** + - A floating-point parameter influencing the merging process, likely related to the timing embedding aspects of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`label_emb.`** + - A floating-point parameter affecting the merging process, possibly related to the label embedding aspects of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i`** + - A series of floating-point parameters for each input block, allowing for fine-tuned control over how these blocks are merged. The index 'i' ranges from 0 to 8, with additional parameters for transformer blocks within certain input blocks. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`middle_block.i`** + - A series of floating-point parameters for each middle block, enabling precise adjustments in the merging of these blocks. The index 'i' ranges from 0 to 2, with additional parameters for transformer blocks within the middle block at index 1. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i`** + - A series of floating-point parameters for each output block, facilitating detailed control over the merging of these blocks. The index 'i' ranges from 0 to 8, with additional parameters for transformer blocks within certain output blocks, based on a reverse index mapping. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`out.`** + - A floating-point parameter that likely influences the final output aspects of the merging process. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The result of the merging process, a single model that combines elements from both input models, with particular attention to transformer blocks. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `GPU` +- Common nodes: unknown + +The ModelMergeSDXLDetailedTransformers node is designed for advanced scenarios where two AI models, particularly those with transformer blocks, need to be merged with high precision. It takes two models as input, along with various parameters to fine-tune the merging of transformer components, and outputs a single, integrated model that combines elements from both input models, focusing on transformer blocks. This node is often used in pipelines requiring the nuanced integration of transformer-based models for tasks like language processing or generative tasks. +## Source code +```python +class ModelMergeSDXLDetailedTransformers(comfy_extras.nodes_model_merging.ModelMergeBlocks): + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + transformers = {4: 2, 5:2, 7:10, 8:10} + transformers_args = ["norm1", "attn1.to_q", "attn1.to_k", "attn1.to_v", "attn1.to_out", "ff.net", "norm2", "attn2.to_q", "attn2.to_k", "attn2.to_v", "attn2.to_out", "norm3"] + + for i in range(9): + arg_dict["input_blocks.{}.0.".format(i)] = argument + if i in transformers: + arg_dict["input_blocks.{}.1.".format(i)] = argument + for j in range(transformers[i]): + for x in transformers_args: + arg_dict["input_blocks.{}.1.transformer_blocks.{}.{}".format(i, j, x)] = argument + + for i in range(3): + arg_dict["middle_block.{}.".format(i)] = argument + if i == 1: + for j in range(10): + for x in transformers_args: + arg_dict["middle_block.{}.transformer_blocks.{}.{}".format(i, j, x)] = argument + + transformers = {3:2, 4: 2, 5:2, 6:10, 7:10, 8:10} + for i in range(9): + arg_dict["output_blocks.{}.0.".format(i)] = argument + t = 8 - i + if t in transformers: + arg_dict["output_blocks.{}.1.".format(i)] = argument + for j in range(transformers[t]): + for x in transformers_args: + arg_dict["output_blocks.{}.1.transformer_blocks.{}.{}".format(i, j, x)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + +``` diff --git a/docs/md/ComfyUI_experiments/ModelMergeSDXLTransformers.md b/docs/md/ComfyUI_experiments/ModelMergeSDXLTransformers.md new file mode 100644 index 0000000000..51c606f79f --- /dev/null +++ b/docs/md/ComfyUI_experiments/ModelMergeSDXLTransformers.md @@ -0,0 +1,112 @@ +# ModelMergeSDXLTransformers +## Documentation +- Class name: `ModelMergeSDXLTransformers` +- Category: `advanced/model_merging` +- Output node: `False` + +This node is designed for merging two models by blending their components, specifically tailored for models with transformer blocks. It extends the functionality of a base model merging class to handle the intricacies of transformer-based models, allowing for detailed control over how different parts of the models are combined. This includes the ability to adjust the blending ratios of various components such as input, middle, and output blocks, as well as specific transformer elements within those blocks. +## Input types +- **`model1`** + - The first model to be merged. It serves as the base model onto which elements from the second model are blended. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`model2`** + - The second model whose elements are to be blended into the first model. This allows for the creation of a new model that inherits characteristics from both input models. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`time_embed.`** + - Adjusts the blending ratio specifically for the time embedding components of the models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`label_emb.`** + - Sets the blending ratio for the label embedding components, allowing for customization of how label information is integrated between the two models. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i.0.`** + - Specifies the blending ratio for the initial part of the input blocks. The 'i' ranges from 0 to 8, enabling customization at the beginning of each input block. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i.1.`** + - Determines the blending ratio for the second part of the input blocks, applicable only to blocks with transformers. This allows for detailed control over the transformer sections within the input blocks. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`input_blocks.i.1.transformer_blocks.j.x`** + - Controls the blending ratio for specific transformer block components within the input blocks, offering granular control over the merging of transformer elements. 'i' ranges from 0 to 8, 'j' corresponds to the transformer block index, and 'x' represents the specific transformer component. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`middle_block.i.`** + - Sets the blending ratio for the middle blocks of the models, enabling precise adjustment of the core parts. 'i' ranges from 0 to 2, providing control over each middle block. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`middle_block.1.transformer_blocks.j.x`** + - Adjusts the blending ratio for specific transformer block components within the middle block, allowing for detailed customization of the transformer elements in the core part of the model. 'j' corresponds to the transformer block index, and 'x' represents the specific transformer component. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i.0.`** + - Specifies the blending ratio for the initial part of the output blocks, enabling customization at the beginning of each output block. 'i' ranges from 0 to 8. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i.1.`** + - Determines the blending ratio for the second part of the output blocks, applicable only to blocks with transformers. This allows for detailed control over the transformer sections within the output blocks. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`output_blocks.i.1.transformer_blocks.j.x`** + - Controls the blending ratio for specific transformer block components within the output blocks, offering granular control over the merging of transformer elements. 'i' ranges from 0 to 8, 'j' corresponds to the transformer block index, and 'x' represents the specific transformer component. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`out.`** + - Sets the overall blending ratio for the models' output. This parameter provides a global control over the final output characteristics of the merged model. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The resulting model after merging the input models according to the specified blending ratios. This model combines elements from both input models, creating a new model with characteristics of both. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +Often used for blending transformer-based models `MODEL1` and `MODEL2` to create a new model that inherits characteristics from both, allowing for detailed customization of blending ratios across various components including input, middle, and output blocks, as well as specific transformer elements within those blocks. +## Source code +```python +class ModelMergeSDXLTransformers(comfy_extras.nodes_model_merging.ModelMergeBlocks): + @classmethod + def INPUT_TYPES(s): + arg_dict = { "model1": ("MODEL",), + "model2": ("MODEL",)} + + argument = ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}) + + arg_dict["time_embed."] = argument + arg_dict["label_emb."] = argument + + transformers = {4: 2, 5:2, 7:10, 8:10} + + for i in range(9): + arg_dict["input_blocks.{}.0.".format(i)] = argument + if i in transformers: + arg_dict["input_blocks.{}.1.".format(i)] = argument + for j in range(transformers[i]): + arg_dict["input_blocks.{}.1.transformer_blocks.{}.".format(i, j)] = argument + + for i in range(3): + arg_dict["middle_block.{}.".format(i)] = argument + if i == 1: + for j in range(10): + arg_dict["middle_block.{}.transformer_blocks.{}.".format(i, j)] = argument + + transformers = {3:2, 4: 2, 5:2, 6:10, 7:10, 8:10} + for i in range(9): + arg_dict["output_blocks.{}.0.".format(i)] = argument + t = 8 - i + if t in transformers: + arg_dict["output_blocks.{}.1.".format(i)] = argument + for j in range(transformers[t]): + arg_dict["output_blocks.{}.1.transformer_blocks.{}.".format(i, j)] = argument + + arg_dict["out."] = argument + + return {"required": arg_dict} + +``` diff --git a/docs/md/ComfyUI_experiments/ModelSamplerTonemapNoiseTest.md b/docs/md/ComfyUI_experiments/ModelSamplerTonemapNoiseTest.md new file mode 100644 index 0000000000..40a99874f1 --- /dev/null +++ b/docs/md/ComfyUI_experiments/ModelSamplerTonemapNoiseTest.md @@ -0,0 +1,66 @@ +# ModelSamplerTonemapNoiseTest +## Documentation +- Class name: `ModelSamplerTonemapNoiseTest` +- Category: `custom_node_experiments` +- Output node: `False` + +This node applies a custom patch to a given model that modifies its sampling behavior. Specifically, it introduces a tonemap noise test by adjusting the magnitude of noise predictions based on the Reinhard tonemapping operator. This adjustment is scaled by a user-defined multiplier, allowing for control over the intensity of the effect. The patched model is then returned, enabling its use in subsequent sampling processes. +## Input types +- **`model`** + - The model to which the custom patch will be applied. This patch modifies the model's sampling behavior to include a tonemap noise test. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`multiplier`** + - A scaling factor that controls the intensity of the tonemap noise effect applied by the patch. It is an optional parameter with a default value of 1.0. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The model after applying the tonemap noise patch. This model incorporates the specified tonemap and noise adjustments, enabling enhanced sampling capabilities. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +Often used with model sampling processes to apply a custom patch that introduces a tonemap noise test, adjusting the noise predictions based on the Reinhard tonemapping operator with a user-defined multiplier for controlling the intensity. This node takes a `MODEL` as input and outputs a patched `MODEL` with enhanced sampling capabilities. +## Source code +```python +class ModelSamplerTonemapNoiseTest: + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "multiplier": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "patch" + + CATEGORY = "custom_node_experiments" + + def patch(self, model, multiplier): + + def sampler_tonemap_reinhard(args): + cond = args["cond"] + uncond = args["uncond"] + cond_scale = args["cond_scale"] + noise_pred = (cond - uncond) + noise_pred_vector_magnitude = (torch.linalg.vector_norm(noise_pred, dim=(1)) + 0.0000000001)[:,None] + noise_pred /= noise_pred_vector_magnitude + + mean = torch.mean(noise_pred_vector_magnitude, dim=(1,2,3), keepdim=True) + std = torch.std(noise_pred_vector_magnitude, dim=(1,2,3), keepdim=True) + + top = (std * 3 + mean) * multiplier + + #reinhard + noise_pred_vector_magnitude *= (1.0 / top) + new_magnitude = noise_pred_vector_magnitude / (noise_pred_vector_magnitude + 1.0) + new_magnitude *= top + + return uncond + noise_pred * new_magnitude * cond_scale + + m = model.clone() + m.set_model_sampler_cfg_function(sampler_tonemap_reinhard) + return (m, ) + +``` diff --git a/docs/md/ComfyUI_experiments/ReferenceOnlySimple.md b/docs/md/ComfyUI_experiments/ReferenceOnlySimple.md new file mode 100644 index 0000000000..9c6f96d868 --- /dev/null +++ b/docs/md/ComfyUI_experiments/ReferenceOnlySimple.md @@ -0,0 +1,86 @@ +# ReferenceOnlySimple +## Documentation +- Class name: `ReferenceOnlySimple` +- Category: `custom_node_experiments` +- Output node: `False` + +This node performs a reference-only operation on a given model by cloning the model and manipulating its attention mechanism based on a reference input. It generates a new latent representation by combining the reference with a batch of zeros, adjusts the attention mechanism to apply specific transformations, and handles optional noise masking for the output. The result is a modified model and a new set of latent samples, potentially with a noise mask, ready for further processing or generation tasks. +## Input types +- **`model`** + - The model to be cloned and modified. This is the primary input for the node, as the operation revolves around altering the model's internal attention mechanism based on the reference input. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`reference`** + - The reference input used to guide the modification of the model's attention mechanism. It provides the basis for generating the new latent representation. + - Python dtype: `Dict[str, torch.Tensor]` + - Comfy dtype: `LATENT` +- **`batch_size`** + - Determines the size of the batch for the new latent representation. It affects the dimensions of the generated zeros tensor that is combined with the reference input. + - Python dtype: `int` + - Comfy dtype: `INT` +## Output types +- **`model`** + - The cloned and modified model with an adjusted attention mechanism. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`latent`** + - A dictionary containing the new set of latent samples and optionally a noise mask, ready for further processing. + - Python dtype: `Dict[str, torch.Tensor]` + - Comfy dtype: `LATENT` +## Usage tips +- Infra type: `GPU` +- Common nodes: `ADE_AnimateDiffLoaderWithContext,LatentFromBatch` + +The ReferenceOnlySimple node is primarily utilized for modifying a given model's attention mechanism by using a reference input to generate a new latent representation, often used with nodes that require a modified model for further processing or generation tasks. It takes a model and a reference latent input, along with an optional batch size, to produce a cloned model with adjusted attention and a new set of latent samples, potentially including a noise mask. +## Source code +```python +class ReferenceOnlySimple: + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "reference": ("LATENT",), + "batch_size": ("INT", {"default": 1, "min": 1, "max": 64}) + }} + + RETURN_TYPES = ("MODEL", "LATENT") + FUNCTION = "reference_only" + + CATEGORY = "custom_node_experiments" + + def reference_only(self, model, reference, batch_size): + model_reference = model.clone() + size_latent = list(reference["samples"].shape) + size_latent[0] = batch_size + latent = {} + latent["samples"] = torch.zeros(size_latent) + + batch = latent["samples"].shape[0] + reference["samples"].shape[0] + def reference_apply(q, k, v, extra_options): + k = k.clone().repeat(1, 2, 1) + offset = 0 + if q.shape[0] > batch: + offset = batch + + for o in range(0, q.shape[0], batch): + for x in range(1, batch): + k[x + o, q.shape[1]:] = q[o,:] + + return q, k, k + + model_reference.set_model_attn1_patch(reference_apply) + out_latent = torch.cat((reference["samples"], latent["samples"])) + if "noise_mask" in latent: + mask = latent["noise_mask"] + else: + mask = torch.ones((64,64), dtype=torch.float32, device="cpu") + + if len(mask.shape) < 3: + mask = mask.unsqueeze(0) + if mask.shape[0] < latent["samples"].shape[0]: + print(latent["samples"].shape, mask.shape) + mask = mask.repeat(latent["samples"].shape[0], 1, 1) + + out_mask = torch.zeros((1,mask.shape[1],mask.shape[2]), dtype=torch.float32, device="cpu") + return (model_reference, {"samples": out_latent, "noise_mask": torch.cat((out_mask, mask))}) + +``` diff --git a/docs/md/ComfyUI_experiments/RescaleClassifierFreeGuidanceTest.md b/docs/md/ComfyUI_experiments/RescaleClassifierFreeGuidanceTest.md new file mode 100644 index 0000000000..2118576ca0 --- /dev/null +++ b/docs/md/ComfyUI_experiments/RescaleClassifierFreeGuidanceTest.md @@ -0,0 +1,60 @@ +# RescaleClassifierFreeGuidanceTest +## Documentation +- Class name: `RescaleClassifierFreeGuidanceTest` +- Category: `custom_node_experiments` +- Output node: `False` + +This method modifies a given model by integrating a custom sampler configuration function. This function recalibrates the classifier-free guidance (CFG) signal, blending the original and modified CFG signals based on a specified multiplier. The adjustment is aimed at refining the model's output, offering a tunable parameter to control the influence of CFG on the generation process. +## Input types +- **`model`** + - The model to be modified by applying a custom sampler configuration function that rescales the classifier-free guidance (CFG) signal based on a multiplier. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`multiplier`** + - A scalar that adjusts the strength of the rescaled CFG signal, blending between the original and rescaled CFG signals. It's a tunable parameter that influences the final output of the model. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The modified model with an integrated custom sampler configuration function for recalibrating the CFG signal. This output is crucial for applying the adjusted CFG signal in the model's generation process. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +Often used to refine the output of AI models by adjusting the strength of the classifier-free guidance signal, making it essential for applications requiring precise control over AI-generated content. It takes a `MODEL` and a `multiplier` as input, and outputs a modified `MODEL` with an integrated custom sampler configuration function for recalibrating the CFG signal. +## Source code +```python +class RescaleClassifierFreeGuidance: + @classmethod + def INPUT_TYPES(s): + return {"required": { "model": ("MODEL",), + "multiplier": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "patch" + + CATEGORY = "custom_node_experiments" + + def patch(self, model, multiplier): + + def rescale_cfg(args): + cond = args["cond"] + uncond = args["uncond"] + cond_scale = args["cond_scale"] + + x_cfg = uncond + cond_scale * (cond - uncond) + ro_pos = torch.std(cond, dim=(1,2,3), keepdim=True) + ro_cfg = torch.std(x_cfg, dim=(1,2,3), keepdim=True) + + x_rescaled = x_cfg * (ro_pos / ro_cfg) + x_final = multiplier * x_rescaled + (1.0 - multiplier) * x_cfg + + return x_final + + m = model.clone() + m.set_model_sampler_cfg_function(rescale_cfg) + return (m, ) + +``` diff --git a/docs/md/ComfyUI_experiments/TonemapNoiseWithRescaleCFG.md b/docs/md/ComfyUI_experiments/TonemapNoiseWithRescaleCFG.md new file mode 100644 index 0000000000..f8cea35691 --- /dev/null +++ b/docs/md/ComfyUI_experiments/TonemapNoiseWithRescaleCFG.md @@ -0,0 +1,81 @@ +# TonemapNoiseWithRescaleCFG +## Documentation +- Class name: `TonemapNoiseWithRescaleCFG` +- Category: `custom_node_experiments` +- Output node: `False` + +This node applies a combination of tonemapping and noise rescaling techniques to modify the behavior of a given model's sampling process. It first calculates the magnitude of the difference between conditional and unconditional predictions, applies a tonemapping operation using the Reinhard method to normalize these magnitudes, and then rescales the conditional guidance scale factor (CFG) based on the standard deviation of the original and modified predictions. This process aims to adjust the model's output in a way that balances detail preservation and noise reduction, potentially enhancing the quality of generated samples. +## Input types +- **`model`** + - The model to be modified. This node clones the provided model and applies the tonemap and rescale operations to its sampling process. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +- **`tonemap_multiplier`** + - A multiplier that adjusts the intensity of the tonemapping operation. Higher values increase the effect of tonemapping on the noise prediction magnitudes. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`rescale_multiplier`** + - A multiplier that adjusts the intensity of the rescaling operation applied to the CFG. It determines the balance between the original and modified predictions in the final output. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +## Output types +- **`model`** + - The modified model with the tonemap and rescale operations applied to its sampling process. This model is expected to produce enhanced samples with a balance between detail preservation and noise reduction. + - Python dtype: `torch.nn.Module` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: `Pack SDXL Tuple` + +The TonemapNoiseWithRescaleCFG node is designed to enhance the quality of AI-generated samples by applying tonemapping and noise rescaling techniques to adjust the model's sampling process. It is particularly useful for models that require a balance between detail preservation and noise reduction, taking a model as input and outputting a modified version of the model with adjusted conditional guidance scale factor (CFG). +## Source code +```python +class TonemapNoiseWithRescaleCFG: + @classmethod + def INPUT_TYPES(s): + return {"required": {"model": ("MODEL",), + "tonemap_multiplier": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 100.0, "step": 0.01}), + "rescale_multiplier": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + }} + RETURN_TYPES = ("MODEL",) + FUNCTION = "patch" + + CATEGORY = "custom_node_experiments" + + def patch(self, model, tonemap_multiplier, rescale_multiplier): + + def tonemap_noise_rescale_cfg(args): + cond = args["cond"] + uncond = args["uncond"] + cond_scale = args["cond_scale"] + + # Tonemap + noise_pred = (cond - uncond) + noise_pred_vector_magnitude = (torch.linalg.vector_norm(noise_pred, dim=(1)) + 0.0000000001)[:, None] + noise_pred /= noise_pred_vector_magnitude + + mean = torch.mean(noise_pred_vector_magnitude, dim=(1, 2, 3), keepdim=True) + std = torch.std(noise_pred_vector_magnitude, dim=(1, 2, 3), keepdim=True) + + top = (std * 3 + mean) * tonemap_multiplier + + # Reinhard + noise_pred_vector_magnitude *= (1.0 / top) + new_magnitude = noise_pred_vector_magnitude / (noise_pred_vector_magnitude + 1.0) + new_magnitude *= top + + # Rescale CFG + x_cfg = uncond + (noise_pred * new_magnitude * cond_scale) + ro_pos = torch.std(cond, dim=(1, 2, 3), keepdim=True) + ro_cfg = torch.std(x_cfg, dim=(1, 2, 3), keepdim=True) + + x_rescaled = x_cfg * (ro_pos / ro_cfg) + x_final = rescale_multiplier * x_rescaled + (1.0 - rescale_multiplier) * x_cfg + + return x_final + + m = model.clone() + m.set_model_sampler_cfg_function(tonemap_noise_rescale_cfg) + return (m, ) + +``` diff --git a/docs/md/SavePNG/PNG_Save.md b/docs/md/SavePNG/PNG_Save.md new file mode 100644 index 0000000000..7eafae4ef5 --- /dev/null +++ b/docs/md/SavePNG/PNG_Save.md @@ -0,0 +1,103 @@ +# PNG Save Node +## Documentation +- Class name: `PNG_Save` +- Category: `Ebsynth Nodes` +- Output node: `True` + +This node processes an image or a batch of images and saves them as PNG files to a specified location. It supports adjusting the image quality, deciding whether to overwrite existing files, and automatically handles the creation of necessary directories. The node utilizes torchvision's encode_png for encoding images. +## Input types +- **`save_path`** + - Specifies the directory where the PNG files will be saved. It defaults to a predefined output directory. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`file_name`** + - The base name for the saved PNG files. A default name is provided, but it can be customized. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`images`** + - The image or batch of images to be saved. The node handles both single images and batches. + - Python dtype: `torch.Tensor` + - Comfy dtype: `IMAGE` +- **`quality`** + - Determines the quality of the saved PNG images. It is adjustable and has a default value set. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`overwrite_mode`** + - Controls whether existing files with the same name should be overwritten. It is enabled by default. + - Python dtype: `bool` + - Comfy dtype: `BOOLEAN` +## Output types +The node doesn't have output types +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +The PNG_Save node is primarily utilized for saving processed images or a batch of images as PNG files to a designated location, with options for adjusting image quality and deciding on file overwrite behavior. It is often used at the end of AI pipelines to output the final image generation, ensuring the results are stored with the desired quality and naming conventions. +## Source code +```python +class PNG_Save: + """ + PNG Save Node + + This node takes in an image and saves it to the specified location using the PNG encoding provided by torchvision's encode_png. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "save_path": ("STRING", {"default": "./ComfyUI/output"}), + "file_name": ("STRING", {"default": "ComfyUI"}), + "images": ("IMAGE",), + "quality": ("INT", {"default": 96, "min": 0, "max": 100, "step": 1}), + "overwrite_mode": ("BOOLEAN", {"default": True}), + } + } + + RETURN_TYPES = () + FUNCTION = "save_image" + OUTPUT_NODE = True + CATEGORY = "Ebsynth Nodes" + + def save_image(self, save_path, file_name, images, quality, overwrite_mode=True): + # If the images tensor has 4 dimensions, it's a batch. Otherwise, make it a batch of 1. + if len(images.shape) == 3: + images = images.unsqueeze(0) + + for idx, image in enumerate(images): + # Construct full path for each image in the batch + base_name, ext = os.path.splitext(file_name) + # Ensure the extension is .png + ext = ".png" + current_file_name = f"{base_name}{ext}" + full_path = os.path.join(save_path, current_file_name) + + # Ensure the directory exists + directory = os.path.dirname(full_path) + os.makedirs(directory, exist_ok=True) + + # Ensure the image tensor has the correct shape + image_tensor = image.permute(2, 0, 1).float() # Change to (C, H, W) format + C, H, W = image_tensor.shape + + if C != 1 and C != 3: + raise ValueError(f"Image tensor has {C} channels. Expected 1 or 3.") + + # Convert and encode the image to PNG + image_tensor = image_tensor.mul(255).clamp(0, 255).type(torch.uint8) + encoded_image = encode_png(image_tensor.cpu()).numpy().tobytes() + + # Check if file exists and overwrite mode is False + if os.path.exists(full_path) and not overwrite_mode: + print(f"File {full_path} already exists and overwrite mode is disabled.") + continue + + # Save the encoded image + with open(full_path, 'wb') as f: + f.write(encoded_image) + + print(f"Image saved to: {full_path}") + + return {"status": "Image(s) saved successfully"} + +``` diff --git a/docs/md/llmnodes/DocumentNode.md b/docs/md/llmnodes/DocumentNode.md new file mode 100644 index 0000000000..43b5193d57 --- /dev/null +++ b/docs/md/llmnodes/DocumentNode.md @@ -0,0 +1,144 @@ +# Document Node +## Documentation +- Class name: `DocumentNode` +- Category: `LLM Nodes` +- Output node: `True` + +This node is responsible for generating documentation for other nodes within a system. It retrieves node information, queries for detailed documentation using a query engine, and saves the documentation in a structured JSON format. This process can be regenerated as needed, ensuring up-to-date documentation. +## Input types +- **`node_query_engine`** + - The engine used to query detailed documentation for nodes. It plays a crucial role in fetching and updating node information based on specific queries. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`query_name`** + - A unique identifier for the query, used to organize and retrieve generated documentation. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`pack_name`** + - The name of the pack containing the node, used to categorize and access nodes within the system. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`node_name`** + - The specific name of the node being documented, used to identify and retrieve node information. + - Python dtype: `str` + - Comfy dtype: `STRING` +- **`regenerate`** + - A flag indicating whether to regenerate the documentation, allowing for updates or corrections. + - Python dtype: `bool` + - Comfy dtype: `BOOLEAN` +## Output types +The node doesn't have output types +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +Primarily utilized for generating and updating documentation of other nodes within a pipeline, the DocumentNode retrieves detailed information about nodes, queries for documentation using a specific engine, and outputs the documentation in a structured JSON format. This node is essential for maintaining up-to-date documentation without manual intervention. +## Source code +```python +class DocumentNode: + @classmethod + def get_all_names(cls): + cls.all_packs = get_all_nodes_packs() + return [ + f"{pack_name}/{node['display_name']}" + for pack_name, pack in cls.all_packs.items() + for node in pack["nodes"].values() + ] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "name": [ + sorted(cls.get_all_names()), + ], + "chunk_lines": ("INT", {"default": 40}), + "chunk_lines_overlap": ("INT", {"default": 15}), + "max_chars": ("INT", {"default": 1500}), + "num_retries": ("INT", {"default": 5}), + "top_k": ("INT", {"default": 10}), + "regenerate": ("BOOLEAN", {"default": False}), + "model": ("MODEL",), + } + } + + RETURN_TYPES = () + OUTPUT_NODE = True + FUNCTION = "document" + CATEGORY = NAME + + def document( + self, + name: str, + chunk_lines: int, + chunk_lines_overlap: int, + max_chars: int, + num_retries: int, + top_k: int, + regenerate: bool, + model: Dict[str, Any], + ) -> Tuple[str,]: + pack_name = name.split("/")[0] + node_display_name = "/".join(name.split("/")[1:]) + for node_name, node_info in self.all_packs[pack_name]["nodes"].items(): + if node_info["display_name"] == node_display_name: + break + emb_name = model["name"].split("_")[0] + f"_{chunk_lines}_{chunk_lines_overlap}_{max_chars}" + query_name = model["name"] + f"_{chunk_lines}_{chunk_lines_overlap}_{max_chars}_{num_retries}_{top_k}" + Settings.llm = model["llm"] + Settings.embed_model = model["embed_model"] + default_chat_func = CondenseQuestionChatEngine.chat + CondenseQuestionChatEngine.chat = custom_chat + index = self._load_index(emb_name, pack_name, chunk_lines, chunk_lines_overlap, max_chars, regenerate) + node_query_engine = NodeQueryEngine(index, num_retries, top_k) + self._document_node(node_query_engine, query_name, pack_name, node_name, regenerate) + Settings.llm = "default" + Settings.embed_model = "default" + CondenseQuestionChatEngine.chat = default_chat_func + return () + + def _load_index( + self, + emb_name: str, + pack_name: str, + chunk_lines: int, + chunk_lines_overlap: int, + max_chars: int, + regenerate: bool, + ) -> VectorStoreIndex: + persist_dir = MAIN_CACHE / "indices" / emb_name / pack_name + if not regenerate and persist_dir.exists() and any(persist_dir.iterdir()): + return load_index_from_storage(StorageContext.from_defaults(persist_dir=persist_dir)) + persist_dir.mkdir(parents=True, exist_ok=True) + # adding comfy files because many packs import from there + code_paths = [self.all_packs[pack_name]["code_path"], self.all_packs["Comfy"]["code_path"]] + code_files = [] + for code_path in code_paths: + code_files += [code_path] if str(code_path).endswith(".py") else list(code_path.rglob("*.py")) + documents = SimpleDirectoryReader(input_files=code_files).load_data() + splitter = CodeSplitter( + language="python", chunk_lines=chunk_lines, chunk_lines_overlap=chunk_lines_overlap, max_chars=max_chars + ) + index = VectorStoreIndex(splitter.get_nodes_from_documents(documents)) + index.storage_context.persist(persist_dir=persist_dir) + return index + + def _document_node( + self, + node_query_engine: NodeQueryEngine, + query_name: str, + pack_name: str, + node_name: str, + regenerate: bool, + ) -> None: + json_file = MAIN_CACHE / "documented_nodes" / query_name / pack_name / f"{node_name}.json" + if not regenerate and json_file.exists(): + with open(json_file, "r") as file: + return file.read() + node_info = self.all_packs[pack_name]["nodes"][node_name] + response = node_query_engine.query(node_name, node_info) + json_file.parent.mkdir(parents=True, exist_ok=True) + with open(json_file, "w") as file: + json.dump(response, file, indent=4) + +``` diff --git a/docs/md/llmnodes/DocumentPack.md b/docs/md/llmnodes/DocumentPack.md new file mode 100644 index 0000000000..d945aca62d --- /dev/null +++ b/docs/md/llmnodes/DocumentPack.md @@ -0,0 +1,83 @@ +# Document Pack +## Documentation +- Class name: `DocumentPack` +- Category: `LLM Nodes` +- Output node: `True` + +The `DocumentPack` node automates the documentation process for a collection of nodes within a specified pack. It leverages a language model to generate comprehensive documentation for each node, ensuring that the documentation is up-to-date and accurately reflects the nodes' functionalities. This process involves indexing node information and customizing chat functions to facilitate effective querying and documentation generation. +## Input types +- **`name`** + - Specifies the name of the node pack to document. This name is crucial as it determines which collection of nodes will be documented, guiding the documentation process. + - Python dtype: `str` + - Comfy dtype: `STRING[COMBO]` +- **`chunk_lines`** + - Determines the chunk size in lines for processing the documentation. This parameter affects how the documentation is segmented, which can influence the comprehensiveness and readability of the generated documentation. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`chunk_lines_overlap`** + - Sets the number of overlapping lines between chunks. This overlap can help ensure continuity and context preservation in the documentation process. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`max_chars`** + - Defines the maximum number of characters for each documentation chunk. This limit helps manage the size of the documentation segments, ensuring they are concise yet informative. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`num_retries`** + - Specifies the number of retries for querying the language model. This parameter can enhance the accuracy and quality of the generated documentation by allowing multiple attempts. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`top_k`** + - Determines the top K responses to consider from the language model queries. This selection process can influence the relevance and quality of the documentation. + - Python dtype: `int` + - Comfy dtype: `INT` +- **`regenerate`** + - Indicates whether to regenerate the documentation. This option allows for updating the documentation to reflect any changes or improvements in the nodes. + - Python dtype: `bool` + - Comfy dtype: `BOOLEAN` +- **`model`** + - Provides the model configuration for documentation generation. This includes settings for the language and embedding models, which are essential for customizing the documentation process. + - Python dtype: `dict` + - Comfy dtype: `MODEL` +## Output types +The node doesn't have output types +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +Utilized primarily for automating the documentation process of a collection of nodes within a specified pack, ensuring up-to-date and accurate reflections of the nodes' functionalities by leveraging a language model. +## Source code +```python +class DocumentPack(DocumentNode): + @classmethod + def get_all_names(cls): + cls.all_packs = get_all_nodes_packs() + return [f"{pack_name}/{len(pack['nodes'])} nodes" for pack_name, pack in cls.all_packs.items()] + + def document( + self, + name: str, + chunk_lines: int, + chunk_lines_overlap: int, + max_chars: int, + num_retries: int, + top_k: int, + regenerate: bool, + model: Dict[str, Any], + ) -> Tuple[str,]: + pack_name = name.split("/")[0] + emb_name = model["name"].split("_")[0] + f"_{chunk_lines}_{chunk_lines_overlap}_{max_chars}" + query_name = model["name"] + f"_{chunk_lines}_{chunk_lines_overlap}_{max_chars}_{num_retries}_{top_k}" + Settings.llm = model["llm"] + Settings.embed_model = model["embed_model"] + default_chat_func = CondenseQuestionChatEngine.chat + CondenseQuestionChatEngine.chat = custom_chat + index = self._load_index(emb_name, pack_name, chunk_lines, chunk_lines_overlap, max_chars, regenerate) + node_query_engine = NodeQueryEngine(index, num_retries, top_k) + for node_name in self.all_packs[pack_name]["nodes"]: + self._document_node(node_query_engine, query_name, pack_name, node_name, regenerate) + Settings.llm = "default" + Settings.embed_model = "default" + CondenseQuestionChatEngine.chat = default_chat_func + return () + +``` diff --git a/docs/md/llmnodes/LoadOpenAIModel.md b/docs/md/llmnodes/LoadOpenAIModel.md new file mode 100644 index 0000000000..d620caf596 --- /dev/null +++ b/docs/md/llmnodes/LoadOpenAIModel.md @@ -0,0 +1,57 @@ +# Load OpenAI Model +## Documentation +- Class name: `LoadOpenAIModel` +- Category: `LLM Nodes` +- Output node: `False` + +This node is designed to facilitate the integration of OpenAI's language models into the user's workflow by loading a specified model and its corresponding embedding model. It also allows for the adjustment of the model's generation behavior through temperature control. The output is a comprehensive object that includes both the language and embedding models, ready for use in various applications. +## Input types +- **`model`** + - Specifies the OpenAI model to be loaded, offering a selection from a comprehensive list of available models. The default setting aims to provide a balance between performance and resource usage. + - Python dtype: `str` + - Comfy dtype: `STRING[COMBO]` +- **`temperature`** + - Controls the randomness of the model's output, allowing for fine-tuning of the generation process. A lower temperature results in more deterministic outputs, while a higher value increases randomness. + - Python dtype: `float` + - Comfy dtype: `FLOAT` +- **`embed_model`** + - Determines the embedding model to be used in conjunction with the language model, enhancing the model's understanding and processing of input data. + - Python dtype: `str` + - Comfy dtype: `STRING[COMBO]` +## Output types +- **`model`** + - A structured object containing the loaded language model, embedding model, and a unique identifier, ready for integration into various applications. + - Python dtype: `Dict[str, Any]` + - Comfy dtype: `MODEL` +## Usage tips +- Infra type: `CPU` +- Common nodes: unknown + +The LoadOpenAIModel node is essential for integrating OpenAI's language models into the pipeline, allowing users to specify a model and its embedding counterpart for enhanced input understanding. It is often used with nodes that require advanced text processing and generation capabilities, providing both the language model and embedding model as output for diverse applications. +## Source code +```python +class LoadOpenAIModel: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model": (sorted(ALL_AVAILABLE_MODELS.keys()), {"default": "gpt-4-turbo-preview"}), + "temperature": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0}), + "embed_model": ( + sorted([x.value for x in OpenAIEmbeddingModelType]), + {"default": "text-embedding-3-small"}, + ), + } + } + + RETURN_TYPES = ("MODEL",) + FUNCTION = "load_openai_model" + CATEGORY = NAME + + def load_openai_model(self, model: str, temperature: int, embed_model: str) -> Dict[str, Any]: + name = f"{model}_{embed_model}_{temperature}" + llm = OpenAI(model=model, temperature=temperature) + embed_model = OpenAIEmbedding(model=embed_model) + return ({"llm": llm, "embed_model": embed_model, "name": name},) + +``` diff --git a/mkdocs.yml b/mkdocs.yml index 047632785a..cde7318c6a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,28 +1,36 @@ site_name: Salt AI Documentation - nav: - Getting Started: index.md - Platform Resources: resources.md - + - Nodes packs documentation: + - ComfyUI_experiments: + - ModelMergeBlockNumber: md/ComfyUI_experiments/ModelMergeBlockNumber.md + - ModelMergeSDXL: md/ComfyUI_experiments/ModelMergeSDXL.md + - ModelMergeSDXLDetailedTransformers: md/ComfyUI_experiments/ModelMergeSDXLDetailedTransformers.md + - ModelMergeSDXLTransformers: md/ComfyUI_experiments/ModelMergeSDXLTransformers.md + - ModelSamplerTonemapNoiseTest: md/ComfyUI_experiments/ModelSamplerTonemapNoiseTest.md + - ReferenceOnlySimple: md/ComfyUI_experiments/ReferenceOnlySimple.md + - RescaleClassifierFreeGuidanceTest: md/ComfyUI_experiments/RescaleClassifierFreeGuidanceTest.md + - TonemapNoiseWithRescaleCFG: md/ComfyUI_experiments/TonemapNoiseWithRescaleCFG.md + - SavePNG: + - PNG_Save: md/SavePNG/PNG_Save.md + - llmnodes: + - DocumentNode: md/llmnodes/DocumentNode.md + - DocumentPack: md/llmnodes/DocumentPack.md + - LoadOpenAIModel: md/llmnodes/LoadOpenAIModel.md theme: name: material - - palette: - - # Palette toggle for light mode - - media: "(prefers-color-scheme: light)" + palette: + - media: '(prefers-color-scheme: light)' scheme: default toggle: icon: material/weather-night name: Switch to dark mode - - # Palette toggle for dark mode - - media: "(prefers-color-scheme: dark)" + - media: '(prefers-color-scheme: dark)' scheme: slate toggle: icon: material/white-balance-sunny name: Switch to light mode - features: - navigation.instant - navigation.tracking @@ -34,35 +42,23 @@ theme: - search.highlight - search.share - content.code.copy - markdown_extensions: - tables - - #codeblocks - pymdownx.highlight: anchor_linenums: true pygments_lang_class: true line_spans: __span - pymdownx.inlinehilite - pymdownx.snippets - - #callouts - admonition - pymdownx.details - pymdownx.superfences - - #definition lists - def_list - - #table of content - toc: permalink: true - - pymdownx.keys - - attr_list - md_in_html - extra: social: - icon: fontawesome/brands/github @@ -70,4 +66,4 @@ extra: name: Salt AI Github - icon: fontawesome/brands/discord link: https://discord.gg/fT2G63zn - name: Salt AI Discord \ No newline at end of file + name: Salt AI Discord