Merge pull request #1372 from elementary-data/ele-2361-show-singular-…

…tests-in-report Added singular tests to groups API.
elementary-data · Mar 3, 2024 · f29b2aa · f29b2aa
2 parents 07d64a9 + 0117224
commit f29b2aa
Show file tree

Hide file tree

Showing 12 changed files with 74 additions and 10 deletions.
diff --git a/elementary/exceptions/exceptions.py b/elementary/exceptions/exceptions.py
@@ -7,10 +7,6 @@
 
 logger = get_logger(__name__)
 
-_QUICKSTART_CLI_ERR_MSG = (
-    "Please refer for guidance - https://docs.elementary-data.com/quickstart-cli"
-)
-
 
 class Error(Exception):
     """Base class for exceptions in this module."""

diff --git a/elementary/monitor/api/groups/groups.py b/elementary/monitor/api/groups/groups.py
@@ -15,14 +15,18 @@
     NormalizedModelSchema,
     NormalizedSourceSchema,
 )
+from elementary.monitor.fetchers.tests.schema import NormalizedTestSchema
 
 FILES_GROUP_KEYWORD = "__files__"
 NO_TAGS_DEFAULT_TREE = "No tags"
 NO_OWNERS_DEFAULT_TREE = "No owners"
 
 
 GROUPABLE_ARTIFACT = Union[
-    NormalizedModelSchema, NormalizedSourceSchema, NormalizedExposureSchema
+    NormalizedModelSchema,
+    NormalizedSourceSchema,
+    NormalizedExposureSchema,
+    NormalizedTestSchema,
 ]
 
 

diff --git a/elementary/monitor/api/models/schema.py b/elementary/monitor/api/models/schema.py
@@ -20,7 +20,7 @@ class NormalizedArtifactSchema(ExtendedBaseModel):
     # Currently, it's model_name to match the CLI UI.
     model_name: str
     normalized_full_path: str
-    fqn: str
+    fqn: Optional[str] = None
 
     @validator("tags", pre=True)
     def load_tags(cls, tags):

diff --git a/elementary/monitor/api/report/report.py b/elementary/monitor/api/report/report.py
@@ -67,9 +67,15 @@ def get_report_data(
             models = models_api.get_models(exclude_elementary_models)
             sources = models_api.get_sources()
             exposures = models_api.get_exposures()
+            tests = tests_api.get_singular_tests()
 
             groups = groups_api.get_groups(
-                artifacts=[*models.values(), *sources.values(), *exposures.values()]
+                artifacts=[
+                    *models.values(),
+                    *sources.values(),
+                    *exposures.values(),
+                    *tests,
+                ]
             )
 
             models_runs = models_api.get_models_runs(

diff --git a/elementary/monitor/api/tests/schema.py b/elementary/monitor/api/tests/schema.py
@@ -48,6 +48,7 @@ class TestMetadataSchema(BaseModel):
     column_name: Optional[str] = None
     test_name: str
     test_display_name: str
+    original_path: str
     latest_run_time: str
     latest_run_time_utc: str
     latest_run_status: str

diff --git a/elementary/monitor/api/tests/tests.py b/elementary/monitor/api/tests/tests.py
@@ -18,7 +18,10 @@
 )
 from elementary.monitor.api.totals_schema import TotalsSchema
 from elementary.monitor.data_monitoring.schema import SelectorFilterSchema
-from elementary.monitor.fetchers.tests.schema import TestResultDBRowSchema
+from elementary.monitor.fetchers.tests.schema import (
+    NormalizedTestSchema,
+    TestResultDBRowSchema,
+)
 from elementary.monitor.fetchers.tests.tests import TestsFetcher
 from elementary.utils.log import get_logger
 from elementary.utils.time import convert_utc_iso_format_to_datetime
@@ -123,6 +126,9 @@ def _get_test_subscribers(test_meta: dict, model_meta: dict) -> List[str]:
             subscribers.append(model_subscribers)
         return subscribers
 
+    def get_singular_tests(self) -> List[NormalizedTestSchema]:
+        return self.tests_fetcher.get_singular_tests()
+
     def get_test_results(
         self,
         invocation_id: Optional[str],
@@ -339,6 +345,7 @@ def _get_test_metadata_from_test_result_db_row(
             column_name=test_result_db_row.column_name,
             test_name=test_result_db_row.test_name,
             test_display_name=test_display_name,
+            original_path=test_result_db_row.original_path,
             latest_run_time=detected_at.isoformat(),
             latest_run_time_utc=detected_at_utc.isoformat(),
             latest_run_status=test_result_db_row.status,

diff --git a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql
@@ -73,6 +73,7 @@
         elementary_test_results.test_alias,
         elementary_test_results.failures,
         elementary_test_results.result_rows,
+        dbt_tests.original_path,
         dbt_tests.meta,
         dbt_tests.tags as test_tags,
         dbt_artifacts.meta as model_meta,

diff --git a/elementary/monitor/dbt_project/macros/get_singular_tests.sql b/elementary/monitor/dbt_project/macros/get_singular_tests.sql
@@ -0,0 +1,14 @@
+{% macro get_singular_tests() %}
+    {% set get_tests_query %}
+        select
+          unique_id,
+          name,
+          original_path,
+          package_name,
+          tags
+        from {{ ref('elementary', 'dbt_tests') }}
+        where type = 'singular'
+    {% endset %}
+    {% set tests_agate = elementary.run_query(get_tests_query) %}
+    {% do return(elementary.agate_to_dicts(tests_agate)) %}
+{% endmacro %}
diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql
@@ -28,6 +28,7 @@
             test_type,
             test_sub_type,
             test_results_description,
+            original_path,
             owners,
             model_owner,
             tags,

diff --git a/elementary/monitor/fetchers/tests/schema.py b/elementary/monitor/fetchers/tests/schema.py
@@ -1,10 +1,16 @@
-from typing import List, Optional, Union
+from typing import List, Literal, Optional, Union
 
+from elementary.monitor.api.models.schema import NormalizedArtifactSchema
 from elementary.utils.pydantic_shim import Field, validator
 from elementary.utils.schema import ExtendedBaseModel
 from elementary.utils.time import convert_partial_iso_format_to_full_iso_format
 
 
+class NormalizedTestSchema(NormalizedArtifactSchema):
+    unique_id: str
+    artifact_type: Literal["test"] = "test"
+
+
 class TestResultDBRowSchema(ExtendedBaseModel):
     __test__ = False  # Mark for pytest - The class name starts with "Test" which throws warnings on pytest runs
 
@@ -22,6 +28,7 @@ class TestResultDBRowSchema(ExtendedBaseModel):
     test_type: str
     test_sub_type: str
     test_results_description: Optional[str]
+    original_path: str
     owners: Optional[List[str]]
     model_owner: Optional[List[str]]
     # tags is a union of test_tags and model_tags that we get from the db.

diff --git a/elementary/monitor/fetchers/tests/tests.py b/elementary/monitor/fetchers/tests/tests.py
@@ -3,7 +3,10 @@
 
 from elementary.clients.dbt.base_dbt_runner import BaseDbtRunner
 from elementary.clients.fetcher.fetcher import FetcherClient
-from elementary.monitor.fetchers.tests.schema import TestResultDBRowSchema
+from elementary.monitor.fetchers.tests.schema import (
+    NormalizedTestSchema,
+    TestResultDBRowSchema,
+)
 from elementary.utils.log import get_logger
 
 logger = get_logger(__name__)
@@ -34,3 +37,21 @@ def get_all_test_results_db_rows(
             TestResultDBRowSchema(**test_result) for test_result in test_results
         ]
         return test_results
+
+    def get_singular_tests(self) -> List[NormalizedTestSchema]:
+        run_operation_response = self.dbt_runner.run_operation(
+            macro_name="elementary_cli.get_singular_tests"
+        )
+        test_results = (
+            json.loads(run_operation_response[0]) if run_operation_response else []
+        )
+        test_results = [
+            NormalizedTestSchema(
+                unique_id=test_result["unique_id"],
+                model_name=test_result["name"],
+                normalized_full_path=f"{test_result['package_name']}/{test_result['original_path']}",
+                tags=test_result["tags"],
+            )
+            for test_result in test_results
+        ]
+        return test_results
diff --git a/tests/mocks/fetchers/tests_fetcher_mock.py b/tests/mocks/fetchers/tests_fetcher_mock.py
@@ -23,6 +23,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="anomaly_detection",
                 test_sub_type="row_count",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners='["Jeff", "Joe"]',
                 tags='["awesome", "awesome-o"]',
                 meta='{ "subscribers": ["@jeff", "joe"], "alert_fields": ["table", "column", "description"] }',
@@ -50,6 +51,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="anomaly_detection",
                 test_sub_type="row_count",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners='["Jeff", "Joe"]',
                 tags='["awesome", "awesome-o"]',
                 meta='{ "subscribers": ["@jeff", "joe"], "alert_fields": ["table", "column", "description"] }',
@@ -77,6 +79,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="anomaly_detection",
                 test_sub_type="freshness",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners='["Joe"]',
                 tags="[]",
                 meta='{ "subscribers": ["@jeff", "joe"], "alert_fields": ["table", "column", "description"] }',
@@ -104,6 +107,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="anomaly_detection",
                 test_sub_type="row_count",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners="[]",
                 tags='["awesome"]',
                 meta='{ "subscribers": ["@jeff", "joe"], "alert_fields": ["table", "column", "description"] }',
@@ -133,6 +137,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="dbt_test",
                 test_sub_type="generic",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners='["Jeff"]',
                 tags='["awesome-o"]',
                 meta="{}",
@@ -160,6 +165,7 @@ def get_all_test_results_db_rows(self, *args, **kwargs):
                 test_type="dbt_test",
                 test_sub_type="generic",
                 test_results_description="This is a fine result",
+                original_path="tests/elementary/tests/test_elementary.py",
                 owners='["Jeff"]',
                 tags='["awesome-o"]',
                 meta="{}",