CodelistPhenotype

Bases: Phenotype

CodelistPhenotype extracts patients from a CodeTable based on a specified codelist and other optional filters such as date range, relative time range and categorical filters.

Parameters:

Name	Type	Description	Default
domain	str	The domain of the phenotype.	required
codelist	Codelist	The codelist used for filtering.	required
name	Optional[str]	The name of the phenotype. Optional. If not passed, name will be derived from the name of the codelist.	None
date_range	DateFilter	A date range filter to apply.	None
relative_time_range	Union[RelativeTimeRangeFilter, List[RelativeTimeRangeFilter]]	A relative time range filter or a list of filters to apply.	None
return_date		Specifies whether to return the 'first', 'last', 'nearest', or 'all' event date(s). Default is 'first'.	'first'
return_value		Specifies which values to return. None for no return value or 'all' for all return values on the selected date(s). Default is None.	None
categorical_filter	Optional[CategoricalFilter]	Additional categorical filters to apply.	None

Attributes:

Name	Type	Description
table	PhenotypeTable	The resulting phenotype table after filtering (None until execute is called)

Examples:

Inpatient Atrial Fibrillation (OMOP)

from phenex.phenotypes import CodelistPhenotype
from phenex.codelists import Codelist
from phenex.mappers import OMOPDomains
from phenex.filters import DateFilter, CategoricalFilter, Value
from phenex.ibis_connect import SnowflakeConnector

con = SnowflakeConnector() # requires some configuration
mapped_tables = OMOPDomains.get_mapped_tables(con)

af_codelist = Codelist([313217]) # list of concept ids
date_range = DateFilter(
    min_date=After("2020-01-01"),
    max_date=Before("2020-12-31")
    )

inpatient = CategoricalFilter(
    column_name='VISIT_DETAIL_CONCEPT_ID',
    allowed_values=[9201],
    domain='VISIT_DETAIL'
)

af_phenotype = CodelistPhenotype(
    name="af",
    domain='CONDITION_OCCURRENCE',
    codelist=af_codelist,
    date_range=date_range,
    return_date='first',
    categorical_filter=inpatient
)

af = af_phenotype.execute(mapped_tables)
af.head()

Myocardial Infarction One Year Pre-index (OMOP)

from phenex.filters import RelativeTimeRangeFilter, Value

af_phenotype = (...) # take from above example

oneyear_preindex = RelativeTimeRangeFilter(
    min_days=Value('>', 0), # exclude index date
    max_days=Value('<', 365),
    anchor_phenotype=af_phenotype # use af phenotype above as reference date
    )

mi_codelist = Codelist([49601007]) # list of concept ids
mi_phenotype = CodelistPhenotype(
    name='mi',
    domain='CONDITION_OCCURRENCE',
    codelist=mi_codelist,
    return_date='first',
    relative_time_range=oneyear_preindex
)
mi = mi_phenotype.execute(mapped_tables)
mi.head()

Source code in phenex/phenotypes/codelist_phenotype.py

class CodelistPhenotype(Phenotype):
    """
    CodelistPhenotype extracts patients from a CodeTable based on a specified codelist and other optional filters such as date range, relative time range and categorical filters.

    Parameters:
        domain: The domain of the phenotype.
        codelist: The codelist used for filtering.
        name: The name of the phenotype. Optional. If not passed, name will be derived from the name of the codelist.
        date_range: A date range filter to apply.
        relative_time_range: A relative time range filter or a list of filters to apply.
        return_date: Specifies whether to return the 'first', 'last', 'nearest', or 'all' event date(s). Default is 'first'.
        return_value: Specifies which values to return. None for no return value or 'all' for all return values on the selected date(s). Default is None.
        categorical_filter: Additional categorical filters to apply.

    Attributes:
        table (PhenotypeTable): The resulting phenotype table after filtering (None until execute is called)

    Examples:

    Example: Inpatient Atrial Fibrillation (OMOP)
        ```python
        from phenex.phenotypes import CodelistPhenotype
        from phenex.codelists import Codelist
        from phenex.mappers import OMOPDomains
        from phenex.filters import DateFilter, CategoricalFilter, Value
        from phenex.ibis_connect import SnowflakeConnector

        con = SnowflakeConnector() # requires some configuration
        mapped_tables = OMOPDomains.get_mapped_tables(con)

        af_codelist = Codelist([313217]) # list of concept ids
        date_range = DateFilter(
            min_date=After("2020-01-01"),
            max_date=Before("2020-12-31")
            )

        inpatient = CategoricalFilter(
            column_name='VISIT_DETAIL_CONCEPT_ID',
            allowed_values=[9201],
            domain='VISIT_DETAIL'
        )

        af_phenotype = CodelistPhenotype(
            name="af",
            domain='CONDITION_OCCURRENCE',
            codelist=af_codelist,
            date_range=date_range,
            return_date='first',
            categorical_filter=inpatient
        )

        af = af_phenotype.execute(mapped_tables)
        af.head()
        ```

    Example: Myocardial Infarction One Year Pre-index (OMOP)
        ```python
        from phenex.filters import RelativeTimeRangeFilter, Value

        af_phenotype = (...) # take from above example

        oneyear_preindex = RelativeTimeRangeFilter(
            min_days=Value('>', 0), # exclude index date
            max_days=Value('<', 365),
            anchor_phenotype=af_phenotype # use af phenotype above as reference date
            )

        mi_codelist = Codelist([49601007]) # list of concept ids
        mi_phenotype = CodelistPhenotype(
            name='mi',
            domain='CONDITION_OCCURRENCE',
            codelist=mi_codelist,
            return_date='first',
            relative_time_range=oneyear_preindex
        )
        mi = mi_phenotype.execute(mapped_tables)
        mi.head()
        ```
    """

    def __init__(
        self,
        domain: str,
        codelist: Codelist,
        name: Optional[str] = None,
        date_range: DateFilter = None,
        relative_time_range: Union[
            RelativeTimeRangeFilter, List[RelativeTimeRangeFilter]
        ] = None,
        return_date="first",
        return_value=None,
        categorical_filter: Optional[CategoricalFilter] = None,
        **kwargs,
    ):
        super(CodelistPhenotype, self).__init__(name=name or codelist.name)

        self.codelist_filter = CodelistFilter(codelist)
        self.codelist = codelist
        self.categorical_filter = categorical_filter
        self.date_range = date_range
        self.return_date = return_date
        self.return_value = return_value
        assert self.return_date in [
            "first",
            "last",
            "nearest",
            "all",
        ], f"Unknown return_date: {return_date}"
        assert self.return_value in [
            None,
            "all",
        ], f"Unknown return_value: {return_value}"
        self.domain = domain
        if isinstance(relative_time_range, RelativeTimeRangeFilter):
            relative_time_range = [relative_time_range]

        self.relative_time_range = relative_time_range
        if self.relative_time_range is not None:
            for rtr in self.relative_time_range:
                if rtr.anchor_phenotype is not None:
                    self.add_children(rtr.anchor_phenotype)

    def _execute(self, tables) -> PhenotypeTable:
        code_table = tables[self.domain]
        code_table = self._perform_codelist_filtering(code_table)
        code_table = self._perform_categorical_filtering(code_table, tables)
        code_table = self._perform_time_filtering(code_table)
        code_table = self._perform_date_selection(code_table)
        code_table = self._perform_value_selection(code_table)
        code_table = select_phenotype_columns(code_table)
        code_table = self._perform_final_processing(code_table)
        return code_table

    def _perform_codelist_filtering(self, code_table):
        assert is_phenex_code_table(code_table)
        code_table = self.codelist_filter.filter(code_table)
        return code_table

    def _perform_categorical_filtering(self, code_table, tables):
        if self.categorical_filter is not None:
            assert is_phenex_code_table(code_table)
            code_table = self.categorical_filter.autojoin_filter(code_table, tables)
        return code_table

    def _perform_time_filtering(self, code_table):
        if self.date_range is not None:
            code_table = self.date_range.filter(code_table)
        if self.relative_time_range is not None:
            for rtr in self.relative_time_range:
                code_table = rtr.filter(code_table)
        return code_table

    def _perform_date_selection(self, code_table):
        """
        Perform date selection based on return_date and return_value parameters.

        Logic:
        - If return_date='all', return all rows (no date aggregation)
        - If return_date='first'/'last'/'nearest' and return_value=None, aggregate to one row per person (reduce=True)
        - If return_date='first'/'last'/'nearest' and return_value='all', keep all rows on the selected date (reduce=False)
        """
        if self.return_date is None or self.return_date == "all":
            return code_table

        # Determine if we should reduce to one row per person or keep all rows on the selected date
        reduce = self.return_value != "all"

        if self.return_date == "first":
            aggregator = First(reduce=reduce)
        elif self.return_date == "last":
            aggregator = Last(reduce=reduce)
        elif self.return_date == "nearest":
            # Note: Nearest is not currently implemented in the aggregators
            # This would need to be added to the aggregator module
            raise NotImplementedError("Nearest aggregation not yet implemented")
        else:
            raise ValueError(f"Unknown return_date: {self.return_date}")

        return aggregator.aggregate(code_table)

    def _perform_value_selection(self, code_table):
        """
        Handle the return_value parameter logic.

        If return_value='all', set the VALUE column to contain the CODE that matched.
        If return_value=None, the VALUE will be set to null by select_phenotype_columns.
        """
        if self.return_value == "all":
            # Set VALUE to the CODE column to return the actual codes that matched
            code_table = code_table.mutate(VALUE=code_table.CODE)

        return code_table

    def get_codelists(self) -> List[Codelist]:
        """
        Get all codelists used in the phenotype definition, including all children / dependent phenotypes.

        Returns:
            codeslist: A list of codelists used in the cohort definition.
        """
        codelists = [self.codelist]
        for p in self.children:
            codelists.extend(p.get_codelists())
        return codelists

dependencies property

Recursively collect all dependencies of a node (including dependencies of dependencies).

Returns:

Type	Description
Set[Node]	List[Node]: A list of Node objects on which this Node depends.

dependency_graph property

Build a dependency graph where each node maps to its direct dependencies (children).

Returns:

Type	Description
Dict[Node, Set[Node]]	Dict[Node, Set[Node]: A mapping of Node's to their children Node's.

execution_metadata property

Retrieve the full execution metadata row for this node from the local DuckDB database.

Returns:

Type	Description
	pandas.Series: A series containing NODE_NAME, LAST_HASH, NODE_PARAMS, and LAST_EXECUTED for this node, or None if the node has never been executed.

namespaced_table property

A PhenotypeTable has generic column names 'person_id', 'boolean', 'event_date', and 'value'. The namespaced_table prepends the phenotype name to all of these columns. This is useful when joining multiple phenotype tables together.

Returns:

Name	Type	Description
table	Table	The namespaced table for the current phenotype.

reverse_dependency_graph property

Build a reverse dependency graph where each node maps to nodes that depend on it (parents).

Returns:

Type	Description
Dict[Node, Set[Node]]	Dict[Node, List[Node]: A mapping of Node's to their parent Node's.

clear_cache(con=None, recursive=False)

Clear the cached state for this node, forcing re-execution on the next call to execute().

This method removes the node's hash from the node states table and optionally drops the materialized table from the database. After calling this method, the node will be treated as if it has never been executed before.

Parameters:

Name	Type	Description	Default
con	Optional[object]	Database connector. If provided, will also drop the materialized table from the database.	None
recursive	bool	If True, also clear the cache for all child nodes recursively. Defaults to False.	False

Example

# Clear cache for a single node
my_node.clear_cache()

# Clear cache and drop materialized table
my_node.clear_cache(con=my_connector)

# Clear cache for node and all its dependencies
my_node.clear_cache(recursive=True)

Source code in phenex/node.py

def clear_cache(self, con: Optional[object] = None, recursive: bool = False):
    """
    Clear the cached state for this node, forcing re-execution on the next call to execute().

    This method removes the node's hash from the node states table and optionally drops the materialized table from the database. After calling this method, the node will be treated as if it has never been executed before.

    Parameters:
        con: Database connector. If provided, will also drop the materialized table from the database.
        recursive: If True, also clear the cache for all child nodes recursively. Defaults to False.

    Example:
        ```python
        # Clear cache for a single node
        my_node.clear_cache()

        # Clear cache and drop materialized table
        my_node.clear_cache(con=my_connector)

        # Clear cache for node and all its dependencies
        my_node.clear_cache(recursive=True)
        ```
    """
    logger.info(f"Node '{self.name}': clearing cached state...")

    # Clear the hash from the node states table
    with Node._hash_update_lock:
        duckdb_con = DuckDBConnector(DUCKDB_DEST_DATABASE=NODE_STATES_DB_NAME)
        if NODE_STATES_TABLE_NAME in duckdb_con.dest_connection.list_tables():
            table = duckdb_con.get_dest_table(NODE_STATES_TABLE_NAME).to_pandas()
            # Remove this node's entry
            table = table[table.NODE_NAME != self.name]

            # Update the table
            if len(table) > 0:
                updated_table = ibis.memtable(table)
                duckdb_con.create_table(
                    updated_table, name_table=NODE_STATES_TABLE_NAME, overwrite=True
                )
            else:
                # Drop the table if it's empty
                duckdb_con.dest_connection.drop_table(NODE_STATES_TABLE_NAME)

    # Drop materialized table if connector is provided
    if con is not None:
        try:
            if self.name in con.dest_connection.list_tables():
                logger.info(f"Node '{self.name}': dropping materialized table...")
                con.dest_connection.drop_table(self.name)
        except Exception as e:
            logger.warning(
                f"Node '{self.name}': failed to drop materialized table: {e}"
            )

    # Reset the table attribute
    self.table = None

    # Recursively clear children if requested
    if recursive:
        for child in self.children:
            child.clear_cache(con=con, recursive=recursive)

    logger.info(f"Node '{self.name}': cache cleared successfully.")

execute(tables=None, con=None, overwrite=False, lazy_execution=False, n_threads=1)

Executes the Node computation for the current node and its dependencies. Supports lazy execution using hash-based change detection to avoid recomputing Node's that have already executed.

Parameters:

Name	Type	Description	Default
tables	Dict[str, Table]	A dictionary mapping domains to Table objects.	None
con	Optional[object]	Connection to database for materializing outputs. If provided, outputs from the node and all children nodes will be materialized (written) to the database using the connector.	None
overwrite	bool	If True, will overwrite any existing tables found in the database while writing. If False, will throw an error when an existing table is found. Has no effect if con is not passed.	False
lazy_execution	bool	If True, only re-executes if the node's definition has changed. Defaults to False. You should pass overwrite=True with lazy_execution as lazy_execution is intended precisely for iterative updates to a node definition. You must pass a connector (to cache results) for lazy_execution to work.	False
n_threads	int	Max number of Node's to execute simultaneously when this node has multiple children.	1

Returns:

Name	Type	Description
Table	Table	The resulting table for this node. Also accessible through self.table after calling self.execute().

Source code in phenex/node.py

def execute(
    self,
    tables: Dict[str, Table] = None,
    con: Optional[object] = None,
    overwrite: bool = False,
    lazy_execution: bool = False,
    n_threads: int = 1,
) -> Table:
    """
    Executes the Node computation for the current node and its dependencies. Supports lazy execution using hash-based change detection to avoid recomputing Node's that have already executed.

    Parameters:
        tables: A dictionary mapping domains to Table objects.
        con: Connection to database for materializing outputs. If provided, outputs from the node and all children nodes will be materialized (written) to the database using the connector.
        overwrite: If True, will overwrite any existing tables found in the database while writing. If False, will throw an error when an existing table is found. Has no effect if con is not passed.
        lazy_execution: If True, only re-executes if the node's definition has changed. Defaults to False. You should pass overwrite=True with lazy_execution as lazy_execution is intended precisely for iterative updates to a node definition. You must pass a connector (to cache results) for lazy_execution to work.
        n_threads: Max number of Node's to execute simultaneously when this node has multiple children.

    Returns:
        Table: The resulting table for this node. Also accessible through self.table after calling self.execute().
    """
    # Handle None tables
    if tables is None:
        tables = {}

    # Build dependency graph for all dependencies
    all_deps = self.dependencies
    nodes = {node.name: node for node in all_deps}
    nodes[self.name] = self  # Add self to the nodes

    # Build dependency and reverse graphs
    dependency_graph = self._build_dependency_graph(nodes)
    reverse_graph = self._build_reverse_graph(dependency_graph)

    # Track completion status and results
    completed = set()
    completion_lock = threading.Lock()
    worker_exceptions = []  # Track exceptions from worker threads
    stop_all_workers = (
        threading.Event()
    )  # Signal to stop all workers on first error

    # Track in-degree for scheduling
    in_degree = {}
    for node_name, dependencies in dependency_graph.items():
        in_degree[node_name] = len(dependencies)
    for node_name in nodes:
        if node_name not in in_degree:
            in_degree[node_name] = 0

    # Queue for nodes ready to execute
    ready_queue = queue.Queue()

    # Add nodes with no dependencies to ready queue
    for node_name, degree in in_degree.items():
        if degree == 0:
            ready_queue.put(node_name)

    def worker():
        """Worker function for thread pool"""
        while not stop_all_workers.is_set():
            try:
                node_name = ready_queue.get(timeout=1)
                # timeout forces to wait 1 second to avoid busy waiting
                if node_name is None:  # Sentinel value to stop worker
                    break
            except queue.Empty:
                continue

            try:
                logger.info(
                    f"Thread {threading.current_thread().name}: executing node '{node_name}'"
                )
                node = nodes[node_name]

                # Execute the node (without recursive child execution since we handle dependencies here)
                if lazy_execution:
                    if not overwrite:
                        raise ValueError(
                            "lazy_execution only works with overwrite=True."
                        )
                    if con is None:
                        raise ValueError(
                            "A DatabaseConnector is required for lazy execution."
                        )

                    if node._get_current_hash() != node._get_last_hash():
                        logger.info(f"Node '{node_name}': computing...")
                        table = node._execute(tables)
                        if (
                            table is not None
                        ):  # Only create table if _execute returns something
                            con.create_table(table, node_name, overwrite=overwrite)
                            table = con.get_dest_table(node_name)
                        node._update_current_hash()
                    else:
                        logger.info(
                            f"Node '{node_name}': unchanged, using cached result"
                        )
                        table = con.get_dest_table(node_name)
                else:
                    table = node._execute(tables)
                    if (
                        con and table is not None
                    ):  # Only create table if _execute returns something
                        con.create_table(table, node_name, overwrite=overwrite)
                        table = con.get_dest_table(node_name)

                node.table = table

                with completion_lock:
                    completed.add(node_name)

                    # Update in-degree for dependent nodes and add ready ones to queue
                    for dependent in reverse_graph.get(node_name, set()):
                        in_degree[dependent] -= 1
                        if in_degree[dependent] == 0:
                            # Check if all dependencies are completed
                            deps_completed = all(
                                dep in completed
                                for dep in dependency_graph.get(dependent, set())
                            )
                            if deps_completed:
                                ready_queue.put(dependent)

                logger.info(
                    f"Thread {threading.current_thread().name}: completed node '{node_name}'"
                )

            except Exception as e:
                logger.error(f"Error executing node '{node_name}': {str(e)}")
                with completion_lock:
                    # Store exception for main thread
                    worker_exceptions.append(e)
                    # Signal all workers to stop immediately and exit worker loop
                    stop_all_workers.set()
                    break
            finally:
                ready_queue.task_done()

    # Start worker threads
    threads = []
    for i in range(min(n_threads, len(nodes))):
        thread = threading.Thread(target=worker, name=f"PhenexWorker-{i}")
        thread.daemon = True
        thread.start()
        threads.append(thread)

    # Wait for all nodes to complete or for an error to occur
    while (
        len(completed) < len(nodes)
        and not worker_exceptions
        and not stop_all_workers.is_set()
    ):
        threading.Event().wait(0.1)  # Small delay to prevent busy waiting

    if not stop_all_workers.is_set():
        # Time to stop workers and cleanup
        stop_all_workers.set()

    # Check if any worker thread had an exception
    if worker_exceptions:
        # Signal workers to stop
        for _ in threads:
            ready_queue.put(None)
        # Wait for threads to finish
        for thread in threads:
            thread.join(timeout=1)
        # Re-raise the first exception
        raise worker_exceptions[0]

    # Signal workers to stop and wait for them
    for _ in threads:
        ready_queue.put(None)  # Sentinel value to stop workers

    for thread in threads:
        thread.join(timeout=1)

    logger.info(
        f"Node '{self.name}': completed multithreaded execution of {len(nodes)} nodes"
    )
    return self.table

get_codelists()

Get all codelists used in the phenotype definition, including all children / dependent phenotypes.

Returns:

Name	Type	Description
codeslist	List[Codelist]	A list of codelists used in the cohort definition.

Source code in phenex/phenotypes/codelist_phenotype.py

def get_codelists(self) -> List[Codelist]:
    """
    Get all codelists used in the phenotype definition, including all children / dependent phenotypes.

    Returns:
        codeslist: A list of codelists used in the cohort definition.
    """
    codelists = [self.codelist]
    for p in self.children:
        codelists.extend(p.get_codelists())
    return codelists

visualize_dependencies()

Create a text visualization of the dependency graph for this node and its dependencies.

Returns:

Name	Type	Description
str	str	A text representation of the dependency graph

Source code in phenex/node.py

def visualize_dependencies(self) -> str:
    """
    Create a text visualization of the dependency graph for this node and its dependencies.

    Returns:
        str: A text representation of the dependency graph
    """
    lines = [f"Dependencies for Node '{self.name}':"]

    # Get all dependencies
    all_deps = self.dependencies
    nodes = {node.name: node for node in all_deps}
    nodes[self.name] = self  # Add self to the nodes

    # Build dependency graph
    dependency_graph = self._build_dependency_graph(nodes)

    for node_name in sorted(nodes.keys()):
        dependencies = dependency_graph.get(node_name, set())
        if dependencies:
            deps_str = ", ".join(sorted(dependencies))
            lines.append(f"  {node_name} depends on: {deps_str}")
        else:
            lines.append(f"  {node_name} (no dependencies)")

    return "\n".join(lines)