Enable usage of pos tags

biaslyze-dev · Aug 25, 2023 · 76723f9 · 76723f9
1 parent ee05ee1
commit 76723f9
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 61 deletions.
diff --git a/biaslyze/_plotly_dashboard.py b/biaslyze/_plotly_dashboard.py
@@ -1,6 +1,6 @@
 """This file contains the new plotting with plotly and dash."""
-from typing import List, Tuple, Dict, Any
 from collections import defaultdict
+from typing import Any, Dict, List, Tuple
 
 import dash
 import numpy as np
@@ -11,9 +11,7 @@
 from plotly.colors import n_colors
 
 
-def _get_default_results(
-    result, concept: str
-) -> pd.DataFrame:
+def _get_default_results(result, concept: str) -> pd.DataFrame:
     """Get the counterfactual scores (default) for each original sample.
 
     Args:
@@ -28,9 +26,7 @@ def _get_default_results(
     return dataf[sort_index.index]
 
 
-def _get_ksr_results(
-    result, concept: str
-) -> pd.DataFrame:
+def _get_ksr_results(result, concept: str) -> pd.DataFrame:
     """Get the counterfactual scores (ksr) for each original sample.
 
     Args:

diff --git a/biaslyze/bias_detectors/counterfactual_biasdetector.py b/biaslyze/bias_detectors/counterfactual_biasdetector.py
@@ -264,27 +264,28 @@ def _extract_counterfactual_concept_samples(
         enumerate(text_representations), total=len(text_representations)
     ):
         present_keywords = concept.get_present_keywords(text_representation)
-        if present_keywords:
-            original_texts.append(text_representation.text)
-            for orig_keyword in present_keywords:
-                counterfactual_texts = concept.get_counterfactual_texts(
-                    orig_keyword,
-                    text_representation,
-                    n_texts=n_texts,
-                    respect_function=respect_function,
-                )
-                for counterfactual_text, counterfactual_keyword in counterfactual_texts:
-                    counterfactual_samples.append(
-                        CounterfactualSample(
-                            text=counterfactual_text,
-                            orig_keyword=orig_keyword.text,
-                            keyword=counterfactual_keyword.text,
-                            concept=concept.name,
-                            tokenized=text_representation,
-                            label=labels[idx] if labels else None,
-                            source_text=text_representation.text,
-                        )
+        if not present_keywords:
+            continue
+        original_texts.append(text_representation.text)
+        for orig_keyword in present_keywords:
+            counterfactual_texts = concept.get_counterfactual_texts(
+                orig_keyword,
+                text_representation,
+                n_texts=n_texts,
+                respect_function=respect_function,
+            )
+            for counterfactual_text, counterfactual_keyword in counterfactual_texts:
+                counterfactual_samples.append(
+                    CounterfactualSample(
+                        text=counterfactual_text,
+                        orig_keyword=orig_keyword.text,
+                        keyword=counterfactual_keyword.text,
+                        concept=concept.name,
+                        tokenized=text_representation,
+                        label=labels[idx] if labels else None,
+                        source_text=text_representation.text,
                     )
+                )
     logger.info(
         f"Extracted {len(counterfactual_samples)} counterfactual sample texts for concept {concept.name} from {len(original_texts)} original texts."
     )

diff --git a/biaslyze/concept_class.py b/biaslyze/concept_class.py
@@ -45,8 +45,7 @@ def can_replace_token(self, token: Token, respect_function: bool = False) -> boo
             bool: True if the keyword can replace the token, False otherwise.
         """
         if respect_function:
-            return True
-            # return token.function in self.functions
+            return token.function in self.functions
         return True
 
     def equal_to_token(self, token: Token) -> bool:

diff --git a/biaslyze/concepts/concepts_en.py b/biaslyze/concepts/concepts_en.py
@@ -1357,16 +1357,16 @@
     "religion": [
         {"keyword": "jew", "function": ["NOUN"], "category": "judaism"},
         {"keyword": "jewish", "function": ["ADJ"], "category": "judaism"},
-        {"keyword": "jews", "function": ["NOUNS"], "category": "judaism"},
+        {"keyword": "jews", "function": ["NOUN"], "category": "judaism"},
         {"keyword": "judaism", "function": ["NOUN"], "category": "judaism"},
         {
             "keyword": "muslim",
             "function": ["NOUN", "ADJ"],
             "category": "islam",
         },
-        {"keyword": "muslims", "function": ["NOUNS"], "category": "islam"},
+        {"keyword": "muslims", "function": ["NOUN"], "category": "islam"},
         {"keyword": "moslem", "function": ["NOUN"], "category": "islam"},
-        {"keyword": "moslems", "function": ["NOUNS"], "category": "islam"},
+        {"keyword": "moslems", "function": ["NOUN"], "category": "islam"},
         {"keyword": "islam", "function": ["NOUN"], "category": "islam"},
         {
             "keyword": "christ",
@@ -1381,14 +1381,14 @@
         {"keyword": "christianity", "function": ["NOUN"], "category": "christianity"},
         {
             "keyword": "christians",
-            "function": ["NOUNS"],
+            "function": ["NOUN"],
             "category": "christianity",
         },
         {"keyword": "buddhism", "function": ["NOUN"], "category": "buddhism"},
         {"keyword": "buddhist", "function": ["NOUN"], "category": "buddhism"},
-        {"keyword": "buddhists", "function": ["NOUNS"], "category": "buddhism"},
+        {"keyword": "buddhists", "function": ["NOUN"], "category": "buddhism"},
         {"keyword": "hindu", "function": ["NOUN"], "category": "hinduism"},
-        {"keyword": "hindus", "function": ["NOUNS"], "category": "hinduism"},
+        {"keyword": "hindus", "function": ["NOUN"], "category": "hinduism"},
         {"keyword": "hinduism", "function": ["NOUN"], "category": "hinduism"},
     ],
     "gender": [
@@ -1397,17 +1397,17 @@
             "function": ["NOUN", "ADJ"],
             "category": "female",
         },
-        {"keyword": "females", "function": ["NOUNS"], "category": "female"},
+        {"keyword": "females", "function": ["NOUN"], "category": "female"},
         {"keyword": "feminine", "function": ["ADJ"], "category": "female"},
         {"keyword": "woman", "function": ["NOUN"], "category": "female"},
-        {"keyword": "women", "function": ["NOUNS"], "category": "female"},
+        {"keyword": "women", "function": ["NOUN"], "category": "female"},
         {"keyword": "she", "function": ["PRON"], "category": "female"},
         {
             "keyword": "her",
             "function": ["PRON", "possessive PRON"],
             "category": "female",
         },
-        {"keyword": "hers", "function": ["possessive PRON"], "category": "female"},
+        {"keyword": "hers", "function": ["PRON"], "category": "female"},
         {"keyword": "herself", "function": ["PRON"], "category": "female"},
         {
             "keyword": "mrs",
@@ -1438,13 +1438,13 @@
         {"keyword": "girl", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "girls",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "mother", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "mothers",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "mom", "function": ["NOUN"], "category": "female"},
@@ -1453,45 +1453,45 @@
         {"keyword": "mummy", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "moms",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "ma", "function": ["NOUN"], "category": "female"},
         {"keyword": "aunt", "function": ["NOUN"], "category": "female"},
         {"keyword": "daughter", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "daughters",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "wife", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "wives",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "grandmother", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "grandmothers",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "grandma", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "grandmas",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "sister", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "sisters",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {"keyword": "sista", "function": ["NOUN"], "category": "female"},
         {
             "keyword": "sistas",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "female",
         },
         {
@@ -1501,7 +1501,7 @@
         },
         {"keyword": "he", "function": ["PRON"], "category": "male"},
         {"keyword": "him", "function": ["PRON"], "category": "male"},
-        {"keyword": "his", "function": ["possessive PRON"], "category": "male"},
+        {"keyword": "his", "function": ["PRON"], "category": "male"},
         {"keyword": "himself", "function": ["PRON"], "category": "male"},
         {"keyword": "mr", "function": ["title", "NOUN"], "category": "male"},
         {"keyword": "sir", "function": ["title", "NOUN"], "category": "male"},
@@ -1527,68 +1527,68 @@
         {"keyword": "guy", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "guys",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "boy", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "boys",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "father", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "fathers",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "dad", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "dads",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "daddy", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "daddies",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "pa", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "pas",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "son", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "sons",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "husband", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "husbands",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "grandfather", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "grandfathers",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "grandpa", "function": ["NOUN"], "category": "male"},
         {
             "keyword": "grandpas",
-            "function": ["possessive NOUN", "NOUNS"],
+            "function": ["NOUN"],
             "category": "male",
         },
         {"keyword": "uncle", "function": ["NOUN"], "category": "male"},
         {"keyword": "they", "function": ["PRON"], "category": "divers"},
         {"keyword": "them", "function": ["PRON"], "category": "divers"},
-        {"keyword": "their", "function": ["possessive PRON"], "category": "divers"},
-        {"keyword": "theirs", "function": ["possessive PRON"], "category": "divers"},
+        {"keyword": "their", "function": ["PRON"], "category": "divers"},
+        {"keyword": "theirs", "function": ["PRON"], "category": "divers"},
         {"keyword": "themself", "function": ["PRON"], "category": "divers"},
         {"keyword": "themselves", "function": ["PRON"], "category": "divers"},
         {

diff --git a/biaslyze/results/__init__.py b/biaslyze/results/__init__.py
@@ -3,4 +3,4 @@
     CounterfactualConceptResult,
     CounterfactualDetectionResult,
     CounterfactualSample,
-)
+)