12. Use embeddings

Use embeddings¶

In this tutorial, we'll use product group embeddings to compare a customer's latest invoice with their past purchases from the last 26 weeks.

To learn how to create a SQL Embedding User-Defined Function (UDF), check out the 'Bring Your Own Transformer' tutorials.

For our hosted tutorials, we have pre-configured a SQL UDF using the SBERT Transformer model on our Snowflake data warehouse. We'll register this UDF in the Catalog and apply it to analyze the ProductGroup descriptions.

In [1]:

import featurebyte as fb

# Set your profile to the tutorial environment
fb.use_profile("tutorial")

catalog_name = "Grocery Dataset Tutorial"
catalog = fb.Catalog.activate(catalog_name)

import featurebyte as fb # Set your profile to the tutorial environment fb.use_profile("tutorial") catalog_name = "Grocery Dataset Tutorial" catalog = fb.Catalog.activate(catalog_name)

15:32:17 | INFO     | SDK version: 1.0.2.dev46
15:32:17 | INFO     | No catalog activated.
15:32:17 | INFO     | Using profile: tutorial
15:32:17 | INFO     | Using configuration file at: /Users/gxav/.featurebyte/config.yaml
15:32:17 | INFO     | Active profile: tutorial (https://tutorials.featurebyte.com/api/v1)
15:32:17 | INFO     | SDK version: 1.0.2.dev46
15:32:17 | INFO     | No catalog activated.
15:32:17 | INFO     | Catalog activated: Grocery Dataset Tutorial

Register the F_SBERT_EMBEDDING UDF to the Catalog¶

In [2]:

fb.UserDefinedFunction.create(
    name='embedding', 
    sql_function_name='F_SBERT_EMBEDDING',
    function_parameters=[fb.FunctionParameter(name="x", dtype=fb.enum.DBVarType.VARCHAR)],
    output_dtype=fb.enum.DBVarType.ARRAY,
    is_global=False,
)

fb.UserDefinedFunction.create( name='embedding', sql_function_name='F_SBERT_EMBEDDING', function_parameters=[fb.FunctionParameter(name="x", dtype=fb.enum.DBVarType.VARCHAR)], output_dtype=fb.enum.DBVarType.ARRAY, is_global=False, )

Out[2]:

User Defined Function

name	embedding
created_at	2024-04-26 07:32:17
updated_at	None
description	None
sql_function_name	F_SBERT_EMBEDDING
function_parameters	name dtype default_value test_value 0 x VARCHAR None None
signature	embedding(x: str) -> array
output_dtype	ARRAY
feature_store_name	playground
used_by_features	[]

Apply the embedding UDF instance to ProductGroup¶

In [3]:

# Get embedding UDF instance.
embedding_udf = catalog.get_user_defined_function(
    "embedding"
)

# Get embedding UDF instance. embedding_udf = catalog.get_user_defined_function( "embedding" )

In [4]:

# Get view from GROCERYPRODUCT dimension table.
groceryproduct_view = catalog.get_view("GROCERYPRODUCT")
# Apply embedding to ProductGroup column in GROCERYPRODUCT view.
groceryproduct_view["ProductGroup_embedding"] = embedding_udf(groceryproduct_view["ProductGroup"])

# Get view from GROCERYPRODUCT dimension table. groceryproduct_view = catalog.get_view("GROCERYPRODUCT") # Apply embedding to ProductGroup column in GROCERYPRODUCT view. groceryproduct_view["ProductGroup_embedding"] = embedding_udf(groceryproduct_view["ProductGroup"])

Get other views¶

In [5]:

# Get view from GROCERYINVOICE event table.
groceryinvoice_view = catalog.get_view("GROCERYINVOICE")
# Get view from INVOICEITEMS item table.
invoiceitems_view = catalog.get_view("INVOICEITEMS")

# Get view from GROCERYINVOICE event table. groceryinvoice_view = catalog.get_view("GROCERYINVOICE") # Get view from INVOICEITEMS item table. invoiceitems_view = catalog.get_view("INVOICEITEMS")

Join views¶

In [6]:

# Join GROCERYPRODUCT view to INVOICEITEMS view.
invoiceitems_view = invoiceitems_view.join(
    groceryproduct_view, rprefix="product_"
)

# Join GROCERYPRODUCT view to INVOICEITEMS view. invoiceitems_view = invoiceitems_view.join( groceryproduct_view, rprefix="product_" )

Get the mean vector of an invoice's Product Group descriptions¶

In [7]:

# Group invoiceitems_view by invoice entity (GroceryInvoiceGuid).
invoiceitems_view_by_invoice =\
invoiceitems_view.groupby("GroceryInvoiceGuid")

# Group invoiceitems_view by invoice entity (GroceryInvoiceGuid). invoiceitems_view_by_invoice =\ invoiceitems_view.groupby("GroceryInvoiceGuid")

In [8]:

# Mean vector of product_ProductGroup_embedding for the invoice.
invoice_mean_vector_of_item_product_productgroup_embedding =\
invoiceitems_view_by_invoice.aggregate(
    "product_ProductGroup_embedding", method=fb.AggFunc.AVG,
    feature_name="INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"
)

# Mean vector of product_ProductGroup_embedding for the invoice. invoice_mean_vector_of_item_product_productgroup_embedding =\ invoiceitems_view_by_invoice.aggregate( "product_ProductGroup_embedding", method=fb.AggFunc.AVG, feature_name="INVOICE_Mean_vector_of_item_product_ProductGroup_embedding" )

Get the mean vector of the Customer's latest invoice¶

In [9]:

# Add INVOICE_Mean_vector_of_item_product_ProductGroup_embedding feature to the GROCERYINVOICE view
# as a column.
groceryinvoice_view =\
groceryinvoice_view.add_feature(
    "INVOICE_Mean_vector_of_item_product_ProductGroup_embedding",
    invoice_mean_vector_of_item_product_productgroup_embedding
)

# Add INVOICE_Mean_vector_of_item_product_ProductGroup_embedding feature to the GROCERYINVOICE view # as a column. groceryinvoice_view =\ groceryinvoice_view.add_feature( "INVOICE_Mean_vector_of_item_product_ProductGroup_embedding", invoice_mean_vector_of_item_product_productgroup_embedding )

In [10]:

# Group GROCERYINVOICE view by customer entity (GroceryCustomerGuid).
groceryinvoice_view_by_customer =\
groceryinvoice_view.groupby(['GroceryCustomerGuid'])

# Group GROCERYINVOICE view by customer entity (GroceryCustomerGuid). groceryinvoice_view_by_customer =\ groceryinvoice_view.groupby(['GroceryCustomerGuid'])

In [11]:

# Get Latest Mean vector of item product_ProductGroup_embedding for the customer
customer_latest_invoice_mean_vector_of_item_product_productgroup_embedding =\
groceryinvoice_view_by_customer.aggregate_over(
    "INVOICE_Mean_vector_of_item_product_ProductGroup_embedding", method="latest",
    feature_names=["CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"],
    windows=[None]
)["CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"]

# Get Latest Mean vector of item product_ProductGroup_embedding for the customer customer_latest_invoice_mean_vector_of_item_product_productgroup_embedding =\ groceryinvoice_view_by_customer.aggregate_over( "INVOICE_Mean_vector_of_item_product_ProductGroup_embedding", method="latest", feature_names=["CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"], windows=[None] )["CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"]

Get the mean vector for the Customer's Product Group descriptions over past 26 weeks¶

In [12]:

# Group INVOICEITEMS view by customer entity (GroceryCustomerGuid).
invoiceitems_view_by_customer =\
invoiceitems_view.groupby(['GroceryCustomerGuid'])

# Group INVOICEITEMS view by customer entity (GroceryCustomerGuid). invoiceitems_view_by_customer =\ invoiceitems_view.groupby(['GroceryCustomerGuid'])

In [13]:

# Get Mean vector of product_ProductGroup_embedding for the customer over time.
feature_group =\
invoiceitems_view_by_customer.aggregate_over(
    "product_ProductGroup_embedding", method="avg",
    feature_names=[
        "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w"
    ],
    windows=["26w"],
)
# Get CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w object from feature group.
customer_mean_vector_of_item_product_productgroup_embedding_26w =\
feature_group["CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w"]

# Get Mean vector of product_ProductGroup_embedding for the customer over time. feature_group =\ invoiceitems_view_by_customer.aggregate_over( "product_ProductGroup_embedding", method="avg", feature_names=[ "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w" ], windows=["26w"], ) # Get CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w object from feature group. customer_mean_vector_of_item_product_productgroup_embedding_26w =\ feature_group["CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w"]

Derive Similarity between latest invoice and 26 weeks purchases¶

In [14]:

# Derive Similarity feature from cosine similarity between
# CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding
# and CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice =\
customer_latest_invoice_mean_vector_of_item_product_productgroup_embedding.vec.cosine_similarity(
    customer_mean_vector_of_item_product_productgroup_embedding_26w
)
# Give a name to new feature
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.name = \
"CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w_vs_latest_invoice"

# Derive Similarity feature from cosine similarity between # CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding # and CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice =\ customer_latest_invoice_mean_vector_of_item_product_productgroup_embedding.vec.cosine_similarity( customer_mean_vector_of_item_product_productgroup_embedding_26w ) # Give a name to new feature customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.name = \ "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w_vs_latest_invoice"

Preview feature¶

In [15]:

# Get observation table: 'Preview Table with 10 items'
preview_table = catalog.get_observation_table("Preview Table with 10 items")

# Get observation table: 'Preview Table with 10 items' preview_table = catalog.get_observation_table("Preview Table with 10 items")

In [16]:

# Preview customer_mean_vector_of_item_product_productgroup_embedding_26w
customer_mean_vector_of_item_product_productgroup_embedding_26w.preview(
    preview_table
)

# Preview customer_mean_vector_of_item_product_productgroup_embedding_26w customer_mean_vector_of_item_product_productgroup_embedding_26w.preview( preview_table )

Out[16]:

	POINT_IN_TIME	GROCERYINVOICEITEMGUID	CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w
0	2022-10-05 14:30:03	fc4769cf-41d6-4fb0-9cfc-fe30502cfa18	[-0.050867025223711004, 0.036183445178904004, ...
1	2022-11-09 18:07:20	37122b82-478b-4d9e-b236-69629b592c0b	[-0.051389772166006, 0.034165915842724, -0.020...
2	2023-04-19 15:25:05	8a60f455-aff1-4b4e-8c63-9ab89df2715d	[-0.052858197418672, 0.028179515303383, -0.013...
3	2022-08-16 15:37:21	6781acc6-652d-4867-a138-2d8adb278886	[-0.055149581778464005, 0.03823821472676, -0.0...
4	2023-03-13 21:11:43	cfc620a2-0054-4b4b-99d5-040cf87cfe2d	[-0.052533533882473005, 0.03202685306233, -0.0...
5	2023-03-19 15:22:18	dcdb7c98-dd62-4287-b432-bfe3a2317ebc	[-0.049488712660913, 0.034617167088656, -0.022...
6	2023-02-27 13:27:47	b807e05c-ff1c-4fb3-a760-e0e8ce29c859	[-0.056006776304074, 0.031965939343047, -0.004...
7	2023-02-15 21:10:22	5daf8edb-8625-4653-aaa0-9ac03df92017	[-0.043588562243401005, 0.022270886707502, -0....
8	2022-11-03 14:39:00	9faf5936-d4bb-4709-a530-c1624ec003a5	[-0.051126038899557004, 0.024472378865799, -0....
9	2022-11-18 10:07:49	ae2ccf38-4e5d-4c76-b1e5-04f12307e45b	[-0.047493995182696, 0.034784211601623004, -0....

In [17]:

# Preview customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.preview(
    preview_table
)

# Preview customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.preview( preview_table )

Out[17]:

	POINT_IN_TIME	GROCERYINVOICEITEMGUID	CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w_vs_latest_invoice
0	2022-11-03 14:39:00	9faf5936-d4bb-4709-a530-c1624ec003a5	0.383319
1	2022-08-16 15:37:21	6781acc6-652d-4867-a138-2d8adb278886	0.657974
2	2022-10-05 14:30:03	fc4769cf-41d6-4fb0-9cfc-fe30502cfa18	0.812872
3	2023-03-19 15:22:18	dcdb7c98-dd62-4287-b432-bfe3a2317ebc	0.885103
4	2022-11-18 10:07:49	ae2ccf38-4e5d-4c76-b1e5-04f12307e45b	0.460290
5	2023-03-13 21:11:43	cfc620a2-0054-4b4b-99d5-040cf87cfe2d	0.611533
6	2022-11-09 18:07:20	37122b82-478b-4d9e-b236-69629b592c0b	0.964328
7	2023-02-15 21:10:22	5daf8edb-8625-4653-aaa0-9ac03df92017	0.776228
8	2023-02-27 13:27:47	b807e05c-ff1c-4fb3-a760-e0e8ce29c859	0.778742
9	2023-04-19 15:25:05	8a60f455-aff1-4b4e-8c63-9ab89df2715d	0.549239

Save feature¶

In [18]:

# Save feature
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.save()

# Save feature customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.save()

Done! |████████████████████████████████████████| 100% in 6.2s (0.16%/s)         

As always, add description and view definition file¶

In [19]:

# Add description
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.update_description(
	"Compare the customer's 4w Mean vector of item "
	"product_ProductGroup_embedding with the customer's most recent "
	"invoice. This comparison is done using the Cosine Similarity metric to"
	" measure how similar these mean vector embeddings are."
)
# See feature definition file
customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.definition

# Add description customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.update_description( "Compare the customer's 4w Mean vector of item " "product_ProductGroup_embedding with the customer's most recent " "invoice. This comparison is done using the Cosine Similarity metric to" " measure how similar these mean vector embeddings are." ) # See feature definition file customer_mean_vector_of_item_product_productgroup_embedding_26w_vs_latest_invoice.definition

Out[19]:

# Generated by SDK version: 1.0.2.dev46
from bson import ObjectId
from featurebyte import DimensionTable
from featurebyte import EventTable
from featurebyte import FeatureJobSetting
from featurebyte import ItemTable
from featurebyte import UserDefinedFunction


# dimension_table name: "GROCERYPRODUCT"
dimension_table = DimensionTable.get_by_id(ObjectId("662b577aaa13c89fa14554e3"))
dimension_view = dimension_table.get_view(
    view_mode="manual", drop_column_names=[], column_cleaning_operations=[]
)
col = dimension_view["ProductGroup"]

# udf_name: embedding, sql_function_name: F_SBERT_EMBEDDING
udf_embedding = UserDefinedFunction.get_by_id(
    ObjectId("662b58815827fd6e8d7545f9")
)
col_1 = udf_embedding(col)
view = dimension_view.copy()
view["ProductGroup_embedding"] = col_1

# item_table name: "INVOICEITEMS", event_table name: "GROCERYINVOICE"
item_table = ItemTable.get_by_id(ObjectId("662b5778aa13c89fa14554e2"))
item_view = item_table.get_view(
    event_suffix=None,
    view_mode="manual",
    drop_column_names=["record_available_at"],
    column_cleaning_operations=[],
    event_drop_column_names=["record_available_at"],
    event_column_cleaning_operations=[],
    event_join_column_names=[
        "Timestamp",
        "GroceryInvoiceGuid",
        "GroceryCustomerGuid",
        "tz_offset",
    ],
)
joined_view = item_view.join(
    view, on="GroceryProductGuid", how="left", rsuffix="", rprefix="product_"
)
feat = joined_view.groupby(
    by_keys=["GroceryInvoiceGuid"], category=None
).aggregate(
    value_column="product_ProductGroup_embedding",
    method="avg",
    feature_name="INVOICE_Mean_vector_of_item_product_ProductGroup_embedding",
    skip_fill_na=True,
)

# event_table name: "GROCERYINVOICE"
event_table = EventTable.get_by_id(ObjectId("662b5775aa13c89fa14554e1"))
event_view = event_table.get_view(
    view_mode="manual",
    drop_column_names=["record_available_at"],
    column_cleaning_operations=[],
)
joined_view_1 = event_view.add_feature(
    new_column_name="INVOICE_Mean_vector_of_item_product_ProductGroup_embedding",
    feature=feat,
    entity_column="GroceryInvoiceGuid",
)
grouped = joined_view_1.groupby(
    by_keys=["GroceryCustomerGuid"], category=None
).aggregate_over(
    value_column="INVOICE_Mean_vector_of_item_product_ProductGroup_embedding",
    method="latest",
    windows=[None],
    feature_names=[
        "CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"
    ],
    feature_job_setting=FeatureJobSetting(
        blind_spot="120s", frequency="3600s", time_modulo_frequency="120s"
    ),
    skip_fill_na=True,
)
feat_1 = grouped[
    "CUSTOMER_Latest_INVOICE_Mean_vector_of_item_product_ProductGroup_embedding"
]
grouped_1 = joined_view.groupby(
    by_keys=["GroceryCustomerGuid"], category=None
).aggregate_over(
    value_column="product_ProductGroup_embedding",
    method="avg",
    windows=["26w"],
    feature_names=[
        "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w"
    ],
    feature_job_setting=FeatureJobSetting(
        blind_spot="120s", frequency="3600s", time_modulo_frequency="120s"
    ),
    skip_fill_na=True,
)
feat_2 = grouped_1[
    "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w"
]
feat_3 = feat_1.vec.cosine_similarity(other=feat_2)
feat_3.name = "CUSTOMER_Mean_vector_of_item_product_ProductGroup_embedding_26w_vs_latest_invoice"
output = feat_3
output.save(_id=ObjectId("662b58875827fd6e8d7545fe"))

Concepts in this tutorial¶

• View Joins
• UDF Transforms
• Feature Transforms

SDK reference for¶

• Get UDF instance
• Add an aggregation by invoice to the event view
• Feature.vec.cosine_similarity()

In [ ]: