Spaces:
Running
Running
| from time import sleep | |
| import streamlit as st | |
| import openai | |
| import pinecone | |
| from postgres_db import query_postgresql_realvest | |
| import numpy as np | |
| PINECONE_API_KEY = st.secrets["PINECONE_API_KEY"] | |
| OPENAI_API_KEY = st.secrets["OPENAI_API_KEY"] | |
| INDEX_NAME = 'realvest-data-v2' | |
| EMBEDDING_MODEL = "text-embedding-ada-002" # OpenAI's best embeddings as of Apr 2023 | |
| MAX_LENGTH_DESC = 200 | |
| MATCH_SCORE_THR = 0.0 | |
| TOP_K = 20 | |
| EMBEDDING_VECTOR_DIM = 1536 | |
| ZERO_EMBEDDING_VECTOR = list(np.zeros(EMBEDDING_VECTOR_DIM)) | |
| def query_pinecone(vector=None, top_k: int=3, include_metadata: bool=True, metadata_filter: dict=None, sleep_time: int=10): | |
| MAX_TRIALS = 5 | |
| trial = 0 | |
| out = None | |
| while (out is None) and (trial < MAX_TRIALS): | |
| try: | |
| out = st.session_state['index'].query(vector=vector, top_k=top_k, filter=metadata_filter, include_metadata=include_metadata) | |
| return out | |
| except pinecone.core.exceptions.PineconeProtocolError as err: | |
| print(f"Error, sleep! {err}") | |
| sleep(sleep_time) | |
| trial = trial + 1 | |
| return out | |
| def sort_dict_by_value(d: dict, ascending: bool=True): | |
| """ | |
| Sort dictionary {k1: v1, k2: v2} by its value. The output is | |
| a sorted list of tuples [(k1, v1), (k2, v2)] | |
| """ | |
| return sorted(d.items(), key=lambda x: x[1], reverse=not ascending) | |
| # initialize connection to pinecone (get API key at app.pinecone.io) | |
| from tenacity import retry, stop_after_attempt, wait_fixed | |
| pinecone.init( | |
| api_key=PINECONE_API_KEY, | |
| environment="us-central1-gcp" # may be different, check at app.pinecone.io | |
| ) | |
| def setup_pinecone_index(): | |
| try: | |
| print("Attempting to set up Pinecone index...") # add this line | |
| if "index" not in st.session_state: | |
| st.session_state['index'] = pinecone.Index(INDEX_NAME) | |
| return st.session_state['index'] | |
| except AttributeError as e: | |
| print("Caught an AttributeError:", e) | |
| raise # Re-raise the exception so that tenacity can catch it and retry | |
| except Exception as e: # add this block | |
| print("Caught an unexpected exception:", e) | |
| raise | |
| def init_session_state(): | |
| try: | |
| st.session_state['index'] = setup_pinecone_index() | |
| # stats = test_pinecone() | |
| except Exception as e: | |
| print("Failed to set up Pinecone index after several attempts. Error:", e) | |
| if 'display_results' not in st.session_state: | |
| st.session_state['display_results'] = False | |
| if 'count_checked' not in st.session_state: | |
| st.session_state['count_checked'] = 0 | |
| def callback_count_checked(): | |
| st.session_state['count_checked'] = 0 | |
| st.session_state['checked_boxes'] = [] | |
| for key in list(st.session_state.keys()): | |
| if (key.split('__')[0] == 'cb_compare') and (st.session_state[key] == True): | |
| st.session_state['count_checked'] += 1 | |
| st.session_state['checked_boxes'].append(key) | |
| def summarize_products(products: list) -> str: | |
| """ | |
| Input: | |
| products = [ | |
| {text information of product#1}, | |
| {text information of product#2}, | |
| {text information of product#3}, | |
| ] | |
| Output: | |
| summary = "{summary of all products}" | |
| """ | |
| NEW_LINE = '\n' | |
| PROMPT_PRODUCTS_SUMMARY = f""" | |
| You are a very sharp and helpful assistant to a group of commercial real estate investors. | |
| You are about to write a summary comparison of a few products whose information are given below: | |
| ----- DESCRIPTION of PRODUCTS ----- | |
| { f"{NEW_LINE*2}---{NEW_LINE*2}".join(products) } | |
| ----------------------------------- | |
| Please write a concise and insightful summary table to compare the products for investors, which should include but not limited to: | |
| - title | |
| - product summary | |
| - category | |
| - asking price | |
| - location | |
| - potential profit margin | |
| and display the resulting table in HTML. | |
| """ | |
| print(f"prompt: {PROMPT_PRODUCTS_SUMMARY}") | |
| openai.api_key = OPENAI_API_KEY | |
| completion = openai.ChatCompletion.create( | |
| model="gpt-4", | |
| messages=[ | |
| {"role": "system", "content": "You are a helpful assistant."}, | |
| {"role": "user", "content": PROMPT_PRODUCTS_SUMMARY} | |
| ] | |
| ) | |
| summary = completion.choices[0].message | |
| return summary | |
| ### Main | |
| # st.set_page_config(layout="centered") | |
| css=''' | |
| <style> | |
| section.main > div {max-width:75rem} | |
| input[type="text"] { | |
| background-color: #F2FEEF !important; | |
| } | |
| </style> | |
| ''' | |
| st.markdown(css, unsafe_allow_html=True) | |
| # remove the hamburger in the upper right hand corner and the Made with Streamlit footer | |
| hide_menu_style = """ | |
| <style> | |
| #MainMenu {visibility: hidden;} | |
| footer {visibility: hidden;} | |
| </style> | |
| """ | |
| st.markdown(hide_menu_style, unsafe_allow_html=True) | |
| # initialize state | |
| init_session_state() | |
| # Create a text input field | |
| #st.markdown('Storages, Car Washes, Laundromats... Ask us anything in your own words 🤗 ') | |
| #query = st.text_input("") | |
| query = st.text_input("Storages, Car Washes, Offices, Laundromats... Ask us anything in your own words 🤗 ") | |
| # Create a button | |
| if st.button('Search'): | |
| # # initialize | |
| # st.session_state.clear() | |
| # init_session_state() | |
| st.session_state['count_checked'] = 0 | |
| st.session_state['checked_boxes'] = [] | |
| ### call OpenAI text-embedding | |
| res = openai.Embedding.create(model=EMBEDDING_MODEL, input=[query], api_key=OPENAI_API_KEY) | |
| xq = res['data'][0]['embedding'] | |
| out = query_pinecone(vector=xq, top_k=TOP_K, include_metadata=True) | |
| if (out is not None) and ('matches' in out): | |
| metadata = {match['metadata']['product_id']: match['metadata'] for match in out['matches'] if 'metadata' in match and match['metadata'] is not None} | |
| ### candidates | |
| metadata = {match['metadata']['product_id']: match['metadata'] for match in out['matches']} | |
| match_score = {match['metadata']['product_id']: match['score'] for match in out['matches']} | |
| above_thr_sorted = [ | |
| item | |
| for item in sort_dict_by_value(match_score, ascending=False) | |
| if item[1] > MATCH_SCORE_THR | |
| ] | |
| pids = metadata.keys() | |
| ### query pids | |
| pids_str = [f"'{pid}'" for pid, _ in above_thr_sorted] | |
| query = f""" | |
| SELECT productid, name, category, alternatename, url, logo, description | |
| FROM main_products | |
| WHERE productid in ({', '.join(pids_str)}); | |
| """ | |
| results = query_postgresql_realvest(query) | |
| results = { | |
| result['productid']: result | |
| for result in results | |
| } | |
| ### For test | |
| # print(f"above_thr_sorted: {above_thr_sorted}") | |
| # print(f"results: {results}") | |
| # print(f"metadata: {metadata}") | |
| # # TEST ONLY | |
| # above_thr_sorted = [('2086773', 0.800059378), ('1951083', 0.797319531), ('1998714', 0.795623)] | |
| # results = {'1951083': {'productid': '1951083', 'name': '2 for 1 Turn-key Business Opportunity in Lynnwood, Washington - BizBuySell', 'category': 'Other', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/2-for-1-Turn-key-Business-Opportunity/1951083/', 'logo': 'https://images.bizbuysell.com/shared/listings/195/1951083/87198e08-a191-4d97-a33b-9e9f40fa02f4-W768.jpg', 'description': 'Your chance to own a successful Korean traditional KBBQ grill restaurant and Korean dive bar. Owner is retiring after 19 years of business. This Korean BBQ restaurant utilizes a traditional grill called "Sot Ttu Kkeong" widely found in Korea. There are 10 separate grilling tables with a unique hood system to eliminate odors immediately. The bar next door may be able to extend hours into the summer. With one shared full kitchen, the new owner will be able to maximize business and potentially earn double income.'}, '1998714': {'productid': '1998714', 'name': 'Portland CPA Firm in Portland, Oregon - BizBuySell', 'category': 'Accounting and Tax Practices', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/Portland-CPA-Firm/1998714/', 'logo': 'https://images.bizbuysell.com/shared/listings/199/1998714/cd02bdb9-32c9-409d-b82e-d0531c12eb39-W768.jpg', 'description': 'OR1002: UPDATED :The seller of this Portland CPA firm is approaching retirement and ready to sell the firm. The firm has a great reputation, has good systems in place, is paperless, and has a great staff. The mix of services offers a consistent stream of cash flow to the owner. The seller is seeking a CPA buyer. The office space is available for continued lease after the sale. Revenues for sale include:7% Accounting, bookkeeping and payroll services26% Income tax preparation services for individual clients35% Income tax preparation services for business and other clients28% Audits and reviews4% Consulting services'}, '2086773': {'productid': '2086773', 'name': 'Asian Grocery Supermarket, 1 owner for 29 years in Salem, Oregon - BizBuySell', 'category': 'Grocery Stores and Supermarkets', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Real-Estate-For-Sale/Asian-Grocery-Supermarket-1-owner-for-29-years/2086773/', 'logo': 'https://images.bizbuysell.com/shared/listings/208/2086773/861f6ba6-a994-4e90-9c62-0a593dae2a31-W768.jpg', 'description': 'Great location, well established and profitable supermarket.We have been the sole owner for almost 29 years, so business boasts of a great reputation.'}} | |
| # metadata = {'2086773': {'asking_price': 1000000.0, 'asking_price_currency': 'USD', 'building_status': 'Established', 'category': 'Grocery Stores and Supermarkets', 'chunk_type': 'profile', 'city': 'Salem', 'document': '# Listing Profile\n \nAsking Price (USD): 1000000 \n\nReason for Selling: Retire ', 'listing_type': 'Retail', 'location': 'Salem, OR', 'main_category': 'Grocery Stores and Supermarkets', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Salem', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '2086773', 'similar_pids': ['2074401', '2087795', '2068650'], 'state_code': 'OR'}, '1951083': {'asking_price': 200000.0, 'asking_price_currency': 'USD', 'category': 'Other', 'chunk_type': 'profile', 'city': 'Lynnwood', 'document': '# Listing Profile\n \nAsking Price (USD): 200000 \n\nReason for Selling: Retiring ', 'location': 'Lynnwood, WA', 'main_category': 'Other', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Lynnwood', 'offers__available_from__address__region': 'Washington', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1951083', 'similar_pids': ['2113741', '2033980', '2034855'], 'state_code': 'WA'}, '1998714': {'asking_price': 900000.0, 'asking_price_currency': 'USD', 'category': 'Accounting and Tax Practices', 'chunk_type': 'profile', 'city': 'Portland', 'document': '# Listing Profile\n \nAsking Price (USD): 900000 \n\nReason for Selling: Approaching retirement ', 'fin__gross_revenue': 958000.0, 'location': 'Portland, OR', 'main_category': 'Accounting and Tax Practices', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Portland', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1998714', 'similar_pids': ['2026155', '2066311'], 'state_code': 'OR'}} | |
| # update | |
| st.session_state['above_thr_sorted'] = above_thr_sorted | |
| st.session_state['results'] = results | |
| st.session_state['metadata'] = metadata | |
| st.session_state['display_results'] = True | |
| else: | |
| print("No matches found.") | |
| metadata = {} | |
| if st.session_state['display_results']: | |
| summary_container = st.empty() | |
| st.divider() | |
| st.header("Results") | |
| # display matched results | |
| for pid, match_score in st.session_state['above_thr_sorted']: | |
| if pid not in st.session_state['results']: | |
| continue | |
| metadata_pid = st.session_state['metadata'][pid] | |
| result = st.session_state['results'][pid] | |
| col_icon, col_info, col_compare = st.columns([2, 6, 1]) | |
| with col_icon: | |
| st.image(result["logo"]) | |
| with col_info: | |
| # TODO: make asking price display $xxx,xxx | |
| st.markdown(f"""match score: { round(100 * match_score, 2) } | |
| <br> | |
| **{result['name']}** | |
| <br> | |
| _Asking Price:_ {metadata_pid.get('asking_price', 'N/A')} | |
| <br> | |
| _Category:_ {metadata_pid.get('category', 'N/A')} | |
| <br> | |
| _Location:_ {metadata_pid.get('location', 'N/A')} | |
| """, unsafe_allow_html=True) | |
| st.markdown(f"""**_Description:_** {result['description'][:MAX_LENGTH_DESC]}...[more]({result['url']}) | |
| """) | |
| with col_compare: | |
| st.checkbox('compare', key=f"cb_compare__{pid}", on_change=callback_count_checked) | |
| # display summary tab | |
| if st.session_state['count_checked'] > 0: | |
| with summary_container.container(): | |
| st.divider() | |
| st.header('Summary') | |
| if st.button('Compare Products'): | |
| # populate pids that are checked | |
| relevant_pids = [key.split('__')[-1] for key in st.session_state['checked_boxes']] | |
| relevant_pids = list(set(relevant_pids)) | |
| # get metadata from pinecone | |
| metadata_filter = { | |
| 'product_id': {"$in": relevant_pids} | |
| } | |
| results = query_pinecone( | |
| vector=ZERO_EMBEDDING_VECTOR, | |
| top_k=100, | |
| include_metadata=True, | |
| metadata_filter=metadata_filter | |
| ) | |
| # organize document by product_id | |
| documents = {} | |
| for res in results['matches']: | |
| pid, chunk_id = res['id'].split('-') | |
| if pid not in documents: | |
| documents[pid] = {} | |
| if "chunk" not in documents[pid]: | |
| documents[pid]['chunk'] = {} | |
| documents[pid]['chunk'][chunk_id] = res['metadata']['document'] | |
| # concatenate documents | |
| products = [] | |
| for pid, doc in documents.items(): | |
| products.append( | |
| doc['chunk']['1'] + '\n\n' + doc['chunk']['2'] | |
| ) | |
| # summarize | |
| with st.spinner('Summarizing...'): | |
| summary = summarize_products(products) | |
| st.markdown(summary.get("content"), unsafe_allow_html=True) | |
| else: | |
| try: | |
| summary_container.empty() | |
| except NameError: | |
| pass | |
| # ### Uncomment if you want to debug states | |
| # with st.expander("developer tool"): | |
| # st.json(st.session_state) | |