Yong persist Pinecone index

.gitignore

@@ -1 +1,103 @@
+########################################################################
+# Python - https://github.com/github/gitignore/blob/master/Python.gitignore
+########################################################################
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+build/
+dist/
+eggs/
+.eggs/
+*.egg-info/
+*.egg
+.prerelease-version
+
+# Unit test / coverage reports
+.coverage
+.coverage\.*
+.pytest_cache/
+.mypy_cache/
+test-reports
+htmlcov
+.hypothesis
+
+# Test fixtures
+cffi_bin
+
+# Pyenv Stuff
+.python-version
+venv
+
+# Autogenerated Protobufs
+lib/streamlit/proto/*_pb2.py
+lib/streamlit/proto/*_pb2.pyi
+frontend/src/lib/proto.js
+frontend/src/lib/proto.d.ts
+
+########################################################################
+# OSX - https://github.com/github/gitignore/blob/master/Global/macOS.gitignore
+########################################################################
+.DS_Store
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+########################################################################
+# node - https://github.com/github/gitignore/blob/master/Node.gitignore
+########################################################################
+# Logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Dependency directories
+node_modules/
+
+# ESLint
+.eslintcache
+
+# Coverage directory used by tools like istanbul
+coverage/
+
+########################################################################
+# Streamlit
+########################################################################
 .streamlit/
+lib/streamlit/static
+streamlit-storage
+
+# Data Files
+**/uber-raw-data-sep14.csv
+
+lib/Pipfile.lock
+
+########################################################################
+# JetBrains
+########################################################################
+.idea
+
+########################################################################
+# Cypress
+########################################################################
+frontend/cypress/downloads
+frontend/cypress/videos
+frontend/cypress/screenshots
+__diff_output__
+cypress.env.json
+frontend/test_results
+
+# Ignore screenshots that don't get used in CircleCI
+frontend/cypress/snapshots/darwin
+frontend/cypress/snapshots/linux/1x
+
+########################################################################
+# VSCode
+########################################################################
+.vscode/

README.md

@@ -16,7 +16,7 @@ git clone https://huggingface.co/spaces/realvest/realvest-app
 pyenv install 3.9
 pyenv local 3.9
 
-poetry use env 3.9.17
+poetry env use 3.9.17
 poetry install
 poetry shell
 ```

app.py

@@ -12,39 +12,26 @@ MAX_LENGTH_DESC = 200
 MATCH_SCORE_THR = 0.0
 TOP_K = 20
 
-def test_pinecone(sleep_time: int=1):
-    MAX_TRIALS = 5
-    trial = 0
-    stats = None
-    while (stats is None) and (trial < MAX_TRIALS):
-        try:
-            print(f"BEFORE: trial: {trial}; stats: {stats}")
-            stats = index.describe_index_stats()
-            print(f"AFTER: trial: {trial}; stats: {stats}")
-            return stats
-        except pinecone.core.exceptions.PineconeProtocolError as err:
-            print(f"Error, sleep! {err}")
-            sleep(sleep_time)
-            trial = trial + 1
-    
-    raise Exception(f'max trials {MAX_TRIALS} Exceeded!')
 
-def query_pinecone(xq, top_k: int=3, include_metadata: bool=True, sleep_time: int=1):
+
+
+
+def query_pinecone(xq, top_k: int=3, include_metadata: bool=True, sleep_time: int=10):
     MAX_TRIALS = 5
     trial = 0
     out = None
     while (out is None) and (trial < MAX_TRIALS):
         try:
             # print(f"BEFORE: trial: {trial}; stats: {out}")
-            out = index.query(xq, top_k=top_k, include_metadata=include_metadata)
+            out = st.session_state['index'].query(xq, top_k=top_k, include_metadata=include_metadata)
             # print(f"AFTER: trial: {trial}; stats: {out}")
             return out
         except pinecone.core.exceptions.PineconeProtocolError as err:
             print(f"Error, sleep! {err}")
             sleep(sleep_time)
             trial = trial + 1
-    
-    raise Exception(f'max trials {MAX_TRIALS} Exceeded!')
+    return out
+    #raise Exception(f'max trials {MAX_TRIALS} Exceeded!')
 
 def sort_dict_by_value(d: dict, ascending: bool=True):
     """
@@ -53,7 +40,37 @@ def sort_dict_by_value(d: dict, ascending: bool=True):
     """
     return sorted(d.items(), key=lambda x: x[1], reverse=not ascending)
 
+# initialize connection to pinecone (get API key at app.pinecone.io)
+from tenacity import retry, stop_after_attempt, wait_fixed
+# initialize connection to pinecone (get API key at app.pinecone.io)
+pinecone.init(
+    api_key=PINECONE_API_KEY,
+    environment="us-central1-gcp"  # may be different, check at app.pinecone.io
+)
+
+
+@retry(stop=stop_after_attempt(5), wait=wait_fixed(15))
+def setup_pinecone_index():
+    try:
+        print("Attempting to set up Pinecone index...")  # add this line
+        if "index" not in st.session_state:
+            st.session_state['index'] = pinecone.Index(INDEX_NAME)
+        return st.session_state['index']
+    except AttributeError as e:
+        print("Caught an AttributeError:", e)
+        raise  # Re-raise the exception so that tenacity can catch it and retry
+    except Exception as e:  # add this block
+        print("Caught an unexpected exception:", e)
+        raise
+
+
+
 def init_session_state():
+    try:
+        st.session_state['index']  = setup_pinecone_index()
+        # stats = test_pinecone()
+    except Exception as e:
+        print("Failed to set up Pinecone index after several attempts. Error:", e)
     if 'display_results' not in st.session_state:
         st.session_state['display_results'] = False
 
@@ -105,14 +122,9 @@ def summarize_products(products: list) -> str:
     summary = completion.choices[0].message
     return summary
 
-# initialize connection to pinecone (get API key at app.pinecone.io)
-pinecone.init(
-    api_key=PINECONE_API_KEY,
-    environment="us-central1-gcp"  # may be different, check at app.pinecone.io
-)
-index = pinecone.Index(INDEX_NAME)
-# stats = test_pinecone()
-# print(f"Pinecone DB stats: {stats}")
+
+
+
 
 ### Main
 # st.set_page_config(layout="centered")
@@ -133,53 +145,60 @@ query = st.text_input("What are you looking for?")
 if st.button('Submit'):
 
     # initialize
-    st.session_state.clear()
-    init_session_state()
+    #st.session_state.clear()
+    #init_session_state()
 
     # ### call OpenAI text-embedding
     res = openai.Embedding.create(model=EMBEDDING_MODEL, input=[query], api_key=OPENAI_API_KEY)
     xq = res['data'][0]['embedding']
     out = query_pinecone(xq, top_k=TOP_K, include_metadata=True)
-
-    ### candidates
-    metadata = {match['metadata']['product_id']: match['metadata'] for match in out['matches']}
-    match_score = {match['metadata']['product_id']: match['score'] for match in out['matches']}
-    above_thr_sorted = [
-        item 
-        for item in sort_dict_by_value(match_score, ascending=False) 
-        if item[1] > MATCH_SCORE_THR
-    ]
-    pids = metadata.keys()
-
-    ### query pids
-    pids_str = [f"'{pid}'" for pid, _ in above_thr_sorted]
-    query = f"""
-    SELECT productid, name, category, alternatename, url, logo, description
-    FROM main_products
-    WHERE productid in ({', '.join(pids_str)});
-    """
-
-    results = query_postgresql_realvest(query)
-    results = {
-        result['productid']: result
-        for result in results
-    }
-
-    ### For test
-    # print(f"above_thr_sorted: {above_thr_sorted}")
-    # print(f"results: {results}")
-    # print(f"metadata: {metadata}")
-    # # TEST ONLY
-    # above_thr_sorted = [('2086773', 0.800059378), ('1951083', 0.797319531), ('1998714', 0.795623)]
-    # results = {'1951083': {'productid': '1951083', 'name': '2 for 1 Turn-key Business Opportunity in Lynnwood, Washington - BizBuySell', 'category': 'Other', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/2-for-1-Turn-key-Business-Opportunity/1951083/', 'logo': 'https://images.bizbuysell.com/shared/listings/195/1951083/87198e08-a191-4d97-a33b-9e9f40fa02f4-W768.jpg', 'description': 'Your chance to own a successful Korean traditional KBBQ grill restaurant and Korean dive bar. Owner is retiring after 19 years of business. This Korean BBQ restaurant utilizes a traditional grill called "Sot Ttu Kkeong" widely found in Korea. There are 10 separate grilling tables with a unique hood system to eliminate odors immediately. The bar next door may be able to extend hours into the summer. With one shared full kitchen, the new owner will be able to maximize business and potentially earn double income.'}, '1998714': {'productid': '1998714', 'name': 'Portland CPA Firm in Portland, Oregon - BizBuySell', 'category': 'Accounting and Tax Practices', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/Portland-CPA-Firm/1998714/', 'logo': 'https://images.bizbuysell.com/shared/listings/199/1998714/cd02bdb9-32c9-409d-b82e-d0531c12eb39-W768.jpg', 'description': 'OR1002: UPDATED :The seller of this Portland CPA firm is approaching retirement and ready to sell the firm. The firm has a great reputation, has good systems in place, is paperless, and has a great staff. The mix of services offers a consistent stream of cash flow to the owner. The seller is seeking a CPA buyer. The office space is available for continued lease after the sale. Revenues for sale include:7% Accounting, bookkeeping and payroll services26% Income tax preparation services for individual clients35% Income tax preparation services for business and other clients28% Audits and reviews4% Consulting services'}, '2086773': {'productid': '2086773', 'name': 'Asian Grocery Supermarket, 1 owner for 29 years in Salem, Oregon - BizBuySell', 'category': 'Grocery Stores and Supermarkets', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Real-Estate-For-Sale/Asian-Grocery-Supermarket-1-owner-for-29-years/2086773/', 'logo': 'https://images.bizbuysell.com/shared/listings/208/2086773/861f6ba6-a994-4e90-9c62-0a593dae2a31-W768.jpg', 'description': 'Great location, well established and profitable supermarket.We have been the sole owner for almost 29 years, so business boasts of a great reputation.'}}
-    # metadata = {'2086773': {'asking_price': 1000000.0, 'asking_price_currency': 'USD', 'building_status': 'Established', 'category': 'Grocery Stores and Supermarkets', 'chunk_type': 'profile', 'city': 'Salem', 'document': '# Listing Profile\n    \nAsking Price (USD): 1000000            \n\nReason for Selling: Retire                                                                ', 'listing_type': 'Retail', 'location': 'Salem, OR', 'main_category': 'Grocery Stores and Supermarkets', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Salem', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '2086773', 'similar_pids': ['2074401', '2087795', '2068650'], 'state_code': 'OR'}, '1951083': {'asking_price': 200000.0, 'asking_price_currency': 'USD', 'category': 'Other', 'chunk_type': 'profile', 'city': 'Lynnwood', 'document': '# Listing Profile\n    \nAsking Price (USD): 200000            \n\nReason for Selling: Retiring                                                                ', 'location': 'Lynnwood, WA', 'main_category': 'Other', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Lynnwood', 'offers__available_from__address__region': 'Washington', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1951083', 'similar_pids': ['2113741', '2033980', '2034855'], 'state_code': 'WA'}, '1998714': {'asking_price': 900000.0, 'asking_price_currency': 'USD', 'category': 'Accounting and Tax Practices', 'chunk_type': 'profile', 'city': 'Portland', 'document': '# Listing Profile\n    \nAsking Price (USD): 900000            \n\nReason for Selling: Approaching retirement                                                                ', 'fin__gross_revenue': 958000.0, 'location': 'Portland, OR', 'main_category': 'Accounting and Tax Practices', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Portland', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1998714', 'similar_pids': ['2026155', '2066311'], 'state_code': 'OR'}}    
-
-    # update
-    st.session_state['above_thr_sorted'] = above_thr_sorted
-    st.session_state['results'] = results
-    st.session_state['metadata'] = metadata
-
-    st.session_state['display_results'] = True
+ 
+    if out is not None and 'matches' in out:
+        metadata = {match['metadata']['product_id']: match['metadata'] for match in out['matches'] if 'metadata' in match and match['metadata'] is not None}
+
+
+        ### candidates
+        metadata = {match['metadata']['product_id']: match['metadata'] for match in out['matches']}
+        match_score = {match['metadata']['product_id']: match['score'] for match in out['matches']}
+        above_thr_sorted = [
+            item 
+            for item in sort_dict_by_value(match_score, ascending=False) 
+            if item[1] > MATCH_SCORE_THR
+        ]
+        pids = metadata.keys()
+
+        ### query pids
+        pids_str = [f"'{pid}'" for pid, _ in above_thr_sorted]
+        query = f"""
+        SELECT productid, name, category, alternatename, url, logo, description
+        FROM main_products
+        WHERE productid in ({', '.join(pids_str)});
+        """
+
+        results = query_postgresql_realvest(query)
+        results = {
+            result['productid']: result
+            for result in results
+        }
+
+        ### For test
+        # print(f"above_thr_sorted: {above_thr_sorted}")
+        # print(f"results: {results}")
+        # print(f"metadata: {metadata}")
+        # # TEST ONLY
+        # above_thr_sorted = [('2086773', 0.800059378), ('1951083', 0.797319531), ('1998714', 0.795623)]
+        # results = {'1951083': {'productid': '1951083', 'name': '2 for 1 Turn-key Business Opportunity in Lynnwood, Washington - BizBuySell', 'category': 'Other', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/2-for-1-Turn-key-Business-Opportunity/1951083/', 'logo': 'https://images.bizbuysell.com/shared/listings/195/1951083/87198e08-a191-4d97-a33b-9e9f40fa02f4-W768.jpg', 'description': 'Your chance to own a successful Korean traditional KBBQ grill restaurant and Korean dive bar. Owner is retiring after 19 years of business. This Korean BBQ restaurant utilizes a traditional grill called "Sot Ttu Kkeong" widely found in Korea. There are 10 separate grilling tables with a unique hood system to eliminate odors immediately. The bar next door may be able to extend hours into the summer. With one shared full kitchen, the new owner will be able to maximize business and potentially earn double income.'}, '1998714': {'productid': '1998714', 'name': 'Portland CPA Firm in Portland, Oregon - BizBuySell', 'category': 'Accounting and Tax Practices', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Opportunity/Portland-CPA-Firm/1998714/', 'logo': 'https://images.bizbuysell.com/shared/listings/199/1998714/cd02bdb9-32c9-409d-b82e-d0531c12eb39-W768.jpg', 'description': 'OR1002: UPDATED :The seller of this Portland CPA firm is approaching retirement and ready to sell the firm. The firm has a great reputation, has good systems in place, is paperless, and has a great staff. The mix of services offers a consistent stream of cash flow to the owner. The seller is seeking a CPA buyer. The office space is available for continued lease after the sale. Revenues for sale include:7% Accounting, bookkeeping and payroll services26% Income tax preparation services for individual clients35% Income tax preparation services for business and other clients28% Audits and reviews4% Consulting services'}, '2086773': {'productid': '2086773', 'name': 'Asian Grocery Supermarket, 1 owner for 29 years in Salem, Oregon - BizBuySell', 'category': 'Grocery Stores and Supermarkets', 'alternatename': None, 'url': 'https://www.bizbuysell.com/Business-Real-Estate-For-Sale/Asian-Grocery-Supermarket-1-owner-for-29-years/2086773/', 'logo': 'https://images.bizbuysell.com/shared/listings/208/2086773/861f6ba6-a994-4e90-9c62-0a593dae2a31-W768.jpg', 'description': 'Great location, well established and profitable supermarket.We have been the sole owner for almost 29 years, so business boasts of a great reputation.'}}
+        # metadata = {'2086773': {'asking_price': 1000000.0, 'asking_price_currency': 'USD', 'building_status': 'Established', 'category': 'Grocery Stores and Supermarkets', 'chunk_type': 'profile', 'city': 'Salem', 'document': '# Listing Profile\n    \nAsking Price (USD): 1000000            \n\nReason for Selling: Retire                                                                ', 'listing_type': 'Retail', 'location': 'Salem, OR', 'main_category': 'Grocery Stores and Supermarkets', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Salem', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '2086773', 'similar_pids': ['2074401', '2087795', '2068650'], 'state_code': 'OR'}, '1951083': {'asking_price': 200000.0, 'asking_price_currency': 'USD', 'category': 'Other', 'chunk_type': 'profile', 'city': 'Lynnwood', 'document': '# Listing Profile\n    \nAsking Price (USD): 200000            \n\nReason for Selling: Retiring                                                                ', 'location': 'Lynnwood, WA', 'main_category': 'Other', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Lynnwood', 'offers__available_from__address__region': 'Washington', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1951083', 'similar_pids': ['2113741', '2033980', '2034855'], 'state_code': 'WA'}, '1998714': {'asking_price': 900000.0, 'asking_price_currency': 'USD', 'category': 'Accounting and Tax Practices', 'chunk_type': 'profile', 'city': 'Portland', 'document': '# Listing Profile\n    \nAsking Price (USD): 900000            \n\nReason for Selling: Approaching retirement                                                                ', 'fin__gross_revenue': 958000.0, 'location': 'Portland, OR', 'main_category': 'Accounting and Tax Practices', 'offer_type': 'Offer', 'offers__available_from__address__locality': 'Portland', 'offers__available_from__address__region': 'Oregon', 'offers__available_from__address__type': 'PostalAddress', 'offers__available_from__type': 'Place', 'product_id': '1998714', 'similar_pids': ['2026155', '2066311'], 'state_code': 'OR'}}    
+
+        # update
+        st.session_state['above_thr_sorted'] = above_thr_sorted
+        st.session_state['results'] = results
+        st.session_state['metadata'] = metadata
+
+        st.session_state['display_results'] = True
+    else:
+        print("No matches found.")
+        metadata = {}
 
 if st.session_state['display_results']: